data_workflow/gfsworkflow.py
def nc_georeference(threddspath, timestamp, region, model):
    """
    Description: Intended to make a THREDDS data server compatible netcdf file out of an incorrectly structured
        netcdf file.
    Author: Riley Hales, 2019
    Dependencies: netCDF4, os, datetime
    see github/rileyhales/datatools for more details
    """
    logging.info('\nProcessing the netCDF files')

    # setting the environment file paths
    netcdfs = os.path.join(threddspath, region, model, timestamp, 'netcdfs')
    processed = os.path.join(threddspath, region, model, timestamp, 'processed')

    # if you already have processed netcdfs files, skip this and quit the function
    if not os.path.exists(netcdfs):
        logging.info('There are no netcdfs to be converted. Skipping netcdf processing.')
        return
    # otherwise, remove anything in the folder before starting (in case there was a partial processing)
    else:
        shutil.rmtree(processed)
        os.mkdir(processed)
        os.chmod(processed, 0o777)

    # list the files that need to be converted
    net_files = os.listdir(netcdfs)
    files = [file for file in net_files if file.endswith('.nc')]
    logging.info('There are ' + str(len(files)) + ' compatible files.')

    # read the first file that we'll copy data from in the next blocks of code
    logging.info('Preparing the reference file')
    path = os.path.join(netcdfs, net_files[0])
    netcdf_obj = netCDF4.Dataset(path, 'r', clobber=False, diskless=True)

    # get a dictionary of the dimensions and their size and rename the north/south and east/west ones
    dimensions = {}
    for dimension in netcdf_obj.dimensions.keys():
        dimensions[dimension] = netcdf_obj.dimensions[dimension].size
    dimensions['lat'] = dimensions['latitude']
    dimensions['lon'] = dimensions['longitude']
    dimensions['time'] = 1
    del dimensions['latitude'], dimensions['longitude']

    # get a list of the variables and remove the one's i'm going to 'manually' correct
    variables = netcdf_obj.variables
    del variables['valid_time'], variables['step'], variables['latitude'], variables['longitude'], variables['surface']
    variables = variables.keys()

    # min lat and lon and the interval between values (these are static values
    netcdf_obj.close()

    # this is where the files start getting copied
    for file in files:
        logging.info('Working on file ' + str(file))
        openpath = os.path.join(netcdfs, file)
        savepath = os.path.join(processed, 'processed_' + file)
        # open the file to be copied
        original = netCDF4.Dataset(openpath, 'r', clobber=False, diskless=True)
        duplicate = netCDF4.Dataset(savepath, 'w', clobber=True, format='NETCDF4', diskless=False)
        # set the global netcdf attributes - important for georeferencing
        duplicate.setncatts(original.__dict__)

        # specify dimensions from what we copied before
        for dimension in dimensions:
            duplicate.createDimension(dimension, dimensions[dimension])

        # 'Manually' create the dimensions that need to be set carefully
        duplicate.createVariable(varname='lat', datatype='f4', dimensions='lat')
        duplicate.createVariable(varname='lon', datatype='f4', dimensions='lon')

        # create the lat and lon values as a 1D array
        duplicate['lat'][:] = original['latitude'][:]
        duplicate['lon'][:] = original['longitude'][:]

        # set the attributes for lat and lon (except fill value, you just can't copy it)
        for attr in original['latitude'].__dict__:
            if attr != "_FillValue":
                duplicate['lat'].setncattr(attr, original['latitude'].__dict__[attr])
        for attr in original['longitude'].__dict__:
            if attr != "_FillValue":
                duplicate['lon'].setncattr(attr, original['longitude'].__dict__[attr])

        # copy the rest of the variables
        hour = 6
        for variable in variables:
            # check to use the lat/lon dimension names
            dimension = original[variable].dimensions
            if 'latitude' in dimension:
                dimension = list(dimension)
                dimension.remove('latitude')
                dimension.append('lat')
                dimension = tuple(dimension)
            if 'longitude' in dimension:
                dimension = list(dimension)
                dimension.remove('longitude')
                dimension.append('lon')
                dimension = tuple(dimension)
            if len(dimension) == 2:
                dimension = ('time', 'lat', 'lon')
            if variable == 'time':
                dimension = ('time',)

            # create the variable
            duplicate.createVariable(varname=variable, datatype='f4', dimensions=dimension)

            # copy the arrays of data and set the timestamp/properties
            date = datetime.datetime.strptime(timestamp, "%Y%m%d%H")
            date = datetime.datetime.strftime(date, "%Y-%m-%d %H:00:00")
            if variable == 'time':
                duplicate[variable][:] = [hour]
                hour = hour + 6
                duplicate[variable].long_name = original[variable].long_name
                duplicate[variable].units = "hours since " + date
                duplicate[variable].axis = "T"
                # also set the begin date of this data
                duplicate[variable].begin_date = timestamp
            if variable == 'lat':
                duplicate[variable][:] = original[variable][:]
                duplicate[variable].axis = "Y"
            if variable == 'lon':
                duplicate[variable][:] = original[variable][:]
                duplicate[variable].axis = "X"
            else:
                duplicate[variable][:] = original[variable][:]
                duplicate[variable].axis = "lat lon"
            duplicate[variable].long_name = original[variable].long_name
            duplicate[variable].begin_date = timestamp
            duplicate[variable].units = original[variable].units

        # close the files, delete the one you just did, start again
        original.close()
        duplicate.sync()
        duplicate.close()

    # delete the netcdfs now that we're done with them triggering future runs to skip this step
    shutil.rmtree(netcdfs)

    logging.info('Finished File Conversions')
    return

    Description: Intended to make a THREDDS data server compatible netcdf file out of an incorrectly structured
        netcdf file.
    Author: Riley Hales, 2019
    Dependencies: netCDF4, os, datetime
    see github/rileyhales/datatools for more details
    
Intended to make a THREDDS data server compatible netcdf file out of an incorrectly structured
netcdf file.
Author: Riley Hales, 2019
Dependencies: netCDF4, os, datetime
see github/rileyhales/datatools for more details
Intended to make a THREDDS data server compatible netcdf file out of an incorrectly structured netcdf file . Author : Riley Hales 2019 Dependencies : netCDF4 os datetime see github / rileyhales / datatools for more details
def nc_georeference(threddspath, timestamp, region, model):
    logging.info('\nProcessing the netCDF files')
    netcdfs = os.path.join(threddspath, region, model, timestamp, 'netcdfs')
    processed = os.path.join(threddspath, region, model, timestamp, 'processed')
    if not os.path.exists(netcdfs):
        logging.info('There are no netcdfs to be converted. Skipping netcdf processing.')
        return
    else:
        shutil.rmtree(processed)
        os.mkdir(processed)
        os.chmod(processed, 0o777)
    net_files = os.listdir(netcdfs)
    files = [file for file in net_files if file.endswith('.nc')]
    logging.info('There are ' + str(len(files)) + ' compatible files.')
    logging.info('Preparing the reference file')
    path = os.path.join(netcdfs, net_files[0])
    netcdf_obj = netCDF4.Dataset(path, 'r', clobber=False, diskless=True)
    dimensions = {}
    for dimension in netcdf_obj.dimensions.keys():
        dimensions[dimension] = netcdf_obj.dimensions[dimension].size
    dimensions['lat'] = dimensions['latitude']
    dimensions['lon'] = dimensions['longitude']
    dimensions['time'] = 1
    del dimensions['latitude'], dimensions['longitude']
    variables = netcdf_obj.variables
    del variables['valid_time'], variables['step'], variables['latitude'], variables['longitude'], variables['surface']
    variables = variables.keys()
    netcdf_obj.close()
    for file in files:
        logging.info('Working on file ' + str(file))
        openpath = os.path.join(netcdfs, file)
        savepath = os.path.join(processed, 'processed_' + file)
        original = netCDF4.Dataset(openpath, 'r', clobber=False, diskless=True)
        duplicate = netCDF4.Dataset(savepath, 'w', clobber=True, format='NETCDF4', diskless=False)
        duplicate.setncatts(original.__dict__)
        for dimension in dimensions:
            duplicate.createDimension(dimension, dimensions[dimension])
        duplicate.createVariable(varname='lat', datatype='f4', dimensions='lat')
        duplicate.createVariable(varname='lon', datatype='f4', dimensions='lon')
        duplicate['lat'][:] = original['latitude'][:]
        duplicate['lon'][:] = original['longitude'][:]
        for attr in original['latitude'].__dict__:
            if attr != "_FillValue":
                duplicate['lat'].setncattr(attr, original['latitude'].__dict__[attr])
        for attr in original['longitude'].__dict__:
            if attr != "_FillValue":
                duplicate['lon'].setncattr(attr, original['longitude'].__dict__[attr])
        hour = 6
        for variable in variables:
            dimension = original[variable].dimensions
            if 'latitude' in dimension:
                dimension = list(dimension)
                dimension.remove('latitude')
                dimension.append('lat')
                dimension = tuple(dimension)
            if 'longitude' in dimension:
                dimension = list(dimension)
                dimension.remove('longitude')
                dimension.append('lon')
                dimension = tuple(dimension)
            if len(dimension) == 2:
                dimension = ('time', 'lat', 'lon')
            if variable == 'time':
                dimension = ('time',)
            duplicate.createVariable(varname=variable, datatype='f4', dimensions=dimension)
            date = datetime.datetime.strptime(timestamp, "%Y%m%d%H")
            date = datetime.datetime.strftime(date, "%Y-%m-%d %H:00:00")
            if variable == 'time':
                duplicate[variable][:] = [hour]
                hour = hour + 6
                duplicate[variable].long_name = original[variable].long_name
                duplicate[variable].units = "hours since " + date
                duplicate[variable].axis = "T"
                duplicate[variable].begin_date = timestamp
            if variable == 'lat':
                duplicate[variable][:] = original[variable][:]
                duplicate[variable].axis = "Y"
            if variable == 'lon':
                duplicate[variable][:] = original[variable][:]
                duplicate[variable].axis = "X"
            else:
                duplicate[variable][:] = original[variable][:]
                duplicate[variable].axis = "lat lon"
            duplicate[variable].long_name = original[variable].long_name
            duplicate[variable].begin_date = timestamp
            duplicate[variable].units = original[variable].units
        original.close()
        duplicate.sync()
        duplicate.close()
    shutil.rmtree(netcdfs)
    logging.info('Finished File Conversions')
    return
def nc_georeference ( threddspath , timestamp , region , model ) : logging . info ( '\nProcessing the netCDF files' ) netcdfs = os . path . join ( threddspath , region , model , timestamp , 'netcdfs' ) processed = os . path . join ( threddspath , region , model , timestamp , 'processed' ) if not os . path . exists ( netcdfs ) : logging . info ( 'There are no netcdfs to be converted. Skipping netcdf processing.' ) return else : shutil . rmtree ( processed ) os . mkdir ( processed ) os . chmod ( processed , 0o777 ) net_files = os . listdir ( netcdfs ) files = [ file for file in net_files if file . endswith ( '.nc' ) ] logging . info ( 'There are ' + str ( len ( files ) ) + ' compatible files.' ) logging . info ( 'Preparing the reference file' ) path = os . path . join ( netcdfs , net_files [ 0 ] ) netcdf_obj = netCDF4 . Dataset ( path , 'r' , clobber = False , diskless = True ) dimensions = { } for dimension in netcdf_obj . dimensions . keys ( ) : dimensions [ dimension ] = netcdf_obj . dimensions [ dimension ] . size dimensions [ 'lat' ] = dimensions [ 'latitude' ] dimensions [ 'lon' ] = dimensions [ 'longitude' ] dimensions [ 'time' ] = 1 del dimensions [ 'latitude' ] , dimensions [ 'longitude' ] variables = netcdf_obj . variables del variables [ 'valid_time' ] , variables [ 'step' ] , variables [ 'latitude' ] , variables [ 'longitude' ] , variables [ 'surface' ] variables = variables . keys ( ) netcdf_obj . close ( ) for file in files : logging . info ( 'Working on file ' + str ( file ) ) openpath = os . path . join ( netcdfs , file ) savepath = os . path . join ( processed , 'processed_' + file ) original = netCDF4 . Dataset ( openpath , 'r' , clobber = False , diskless = True ) duplicate = netCDF4 . Dataset ( savepath , 'w' , clobber = True , format = 'NETCDF4' , diskless = False ) duplicate . setncatts ( original . __dict__ ) for dimension in dimensions : duplicate . createDimension ( dimension , dimensions [ dimension ] ) duplicate . createVariable ( varname = 'lat' , datatype = 'f4' , dimensions = 'lat' ) duplicate . createVariable ( varname = 'lon' , datatype = 'f4' , dimensions = 'lon' ) duplicate [ 'lat' ] [ : ] = original [ 'latitude' ] [ : ] duplicate [ 'lon' ] [ : ] = original [ 'longitude' ] [ : ] for attr in original [ 'latitude' ] . __dict__ : if attr != "_FillValue" : duplicate [ 'lat' ] . setncattr ( attr , original [ 'latitude' ] . __dict__ [ attr ] ) for attr in original [ 'longitude' ] . __dict__ : if attr != "_FillValue" : duplicate [ 'lon' ] . setncattr ( attr , original [ 'longitude' ] . __dict__ [ attr ] ) hour = 6 for variable in variables : dimension = original [ variable ] . dimensions if 'latitude' in dimension : dimension = list ( dimension ) dimension . remove ( 'latitude' ) dimension . append ( 'lat' ) dimension = tuple ( dimension ) if 'longitude' in dimension : dimension = list ( dimension ) dimension . remove ( 'longitude' ) dimension . append ( 'lon' ) dimension = tuple ( dimension ) if len ( dimension ) == 2 : dimension = ( 'time' , 'lat' , 'lon' ) if variable == 'time' : dimension = ( 'time' , ) duplicate . createVariable ( varname = variable , datatype = 'f4' , dimensions = dimension ) date = datetime . datetime . strptime ( timestamp , "%Y%m%d%H" ) date = datetime . datetime . strftime ( date , "%Y-%m-%d %H:00:00" ) if variable == 'time' : duplicate [ variable ] [ : ] = [ hour ] hour = hour + 6 duplicate [ variable ] . long_name = original [ variable ] . long_name duplicate [ variable ] . units = "hours since " + date duplicate [ variable ] . axis = "T" duplicate [ variable ] . begin_date = timestamp if variable == 'lat' : duplicate [ variable ] [ : ] = original [ variable ] [ : ] duplicate [ variable ] . axis = "Y" if variable == 'lon' : duplicate [ variable ] [ : ] = original [ variable ] [ : ] duplicate [ variable ] . axis = "X" else : duplicate [ variable ] [ : ] = original [ variable ] [ : ] duplicate [ variable ] . axis = "lat lon" duplicate [ variable ] . long_name = original [ variable ] . long_name duplicate [ variable ] . begin_date = timestamp duplicate [ variable ] . units = original [ variable ] . units original . close ( ) duplicate . sync ( ) duplicate . close ( ) shutil . rmtree ( netcdfs ) logging . info ( 'Finished File Conversions' ) return
Description: Intended to make a THREDDS data server compatible netcdf file out of an incorrectly structured
netcdf file.
Description : Intended to make a THREDDS data server compatible netcdf file out of an incorrectly structured netcdf file .
"""
    Description: Intended to make a THREDDS data server compatible netcdf file out of an incorrectly structured
        netcdf file.
    Author: Riley Hales, 2019
    Dependencies: netCDF4, os, datetime
    see github/rileyhales/datatools for more details
    """ # setting the environment file paths # if you already have processed netcdfs files, skip this and quit the function # otherwise, remove anything in the folder before starting (in case there was a partial processing) # list the files that need to be converted # read the first file that we'll copy data from in the next blocks of code # get a dictionary of the dimensions and their size and rename the north/south and east/west ones # get a list of the variables and remove the one's i'm going to 'manually' correct # min lat and lon and the interval between values (these are static values # this is where the files start getting copied # open the file to be copied # set the global netcdf attributes - important for georeferencing # specify dimensions from what we copied before # 'Manually' create the dimensions that need to be set carefully # create the lat and lon values as a 1D array # set the attributes for lat and lon (except fill value, you just can't copy it) # copy the rest of the variables # check to use the lat/lon dimension names # create the variable # copy the arrays of data and set the timestamp/properties # also set the begin date of this data # close the files, delete the one you just did, start again # delete the netcdfs now that we're done with them triggering future runs to skip this step
threddspath
timestamp
region
model