data_workflow/wrfprworkflow.py
def wrfpr_tiffs(threddspath, wrksppath, timestamp, region):
    """
    Script to convert grib files with multiple variables to Total Accumulated Precipitation GeoTIFFs.
    Dependencies: datetime, os, shutil, numpy, rasterio
    """
    logging.info('\nStarting to process the WRF-PR gribs into GeoTIFFs')
    # declare the environment
    tiffs = os.path.join(wrksppath, region, 'wrfpr_GeoTIFFs')
    gribs = os.path.join(threddspath, region, 'wrfpr', timestamp, 'gribs')
    netcdfs = os.path.join(threddspath, region, 'wrfpr', timestamp, 'netcdfs')

    # if you already have wrf-pr netcdfs in the netcdfs folder, quit the function
    if not os.path.exists(gribs):
        logging.info('There is no gribs folder, you must have already run this step. Skipping conversions')
        return
    # otherwise, remove anything in the folder before starting (in case there was a partial conversion)
    else:
        shutil.rmtree(netcdfs)
        os.mkdir(netcdfs)
        os.chmod(netcdfs, 0o777)
        shutil.rmtree(tiffs)
        os.mkdir(tiffs)
        os.chmod(tiffs, 0o777)

    # create a list of all the files of type "grb" and convert to a list of their file paths
    files = os.listdir(gribs)
    files = [grib for grib in files if grib.endswith('.grb')]
    files.sort()

    # Read raster dimensions only once to apply to all rasters
    path = os.path.join(gribs, files[0])
    raster_dim = rasterio.open(path)
    width = raster_dim.width
    height = raster_dim.height
    lon_min = raster_dim.bounds.left
    lon_max = raster_dim.bounds.right
    lat_min = raster_dim.bounds.bottom
    lat_max = raster_dim.bounds.top

    # Geotransform for each 24-hr raster (east, south, west, north, width, height)
    geotransform = rasterio.transform.from_bounds(lon_min, lat_min, lon_max, lat_max, width, height)

    # Create 1-hr GeoTIFFs and NetCDFs
    for i, j in zip(files, range(len(files))):
        logging.info('working on file ' + i)
        if j == 0:
            path = os.path.join(gribs, i)
            src = rasterio.open(path)
            file_array = src.read(1)
        else:
            cum_file = os.path.join(gribs, i)
            cum_src = rasterio.open(cum_file)
            cum_array = cum_src.read(1)

            past_file = os.path.join(gribs, files[j-1])
            past_src = rasterio.open(past_file)
            past_array = past_src.read(1)

            file_array = cum_array - past_array

        # using the last grib file for the day (path) convert it to a netcdf and set the variable to file_array
        logging.info('opening grib file ' + path)
        obj = xarray.open_dataset(path, engine='cfgrib', backend_kwargs={'filter_by_keys': {'typeOfLevel': 'surface'}})
        logging.info('converting it to a netcdf')
        ncname = i.replace('.grb', '.nc')
        logging.info('saving it to the path ' + path)
        ncpath = os.path.join(netcdfs, ncname)
        obj.to_netcdf(ncpath, mode='w')
        logging.info('converted')
        logging.info('writing the correct values to the tp array')
        nc = netCDF4.Dataset(ncpath, 'a')
        nc['tp'][:] = file_array
        nc.close()
        logging.info('created a netcdf')

        # Specify the GeoTIFF filepath
        tif_filename = i.replace('grb', 'tif')
        tif_filepath = os.path.join(tiffs, tif_filename)

        # Save the 24-hr raster
        with rasterio.open(
                tif_filepath,
                'w',
                driver='GTiff',
                height=file_array.shape[0],
                width=file_array.shape[1],
                count=1,
                dtype=file_array.dtype,
                nodata=numpy.nan,
                crs='+proj=latlong',
                transform=geotransform,
        ) as dst:
            dst.write(file_array, 1)
        logging.info('wrote it to a GeoTIFF\n')

    # # clear the gribs folder now that we're done with this
    shutil.rmtree(gribs)

    return

    Script to convert grib files with multiple variables to Total Accumulated Precipitation GeoTIFFs.
    Dependencies: datetime, os, shutil, numpy, rasterio
    
Script to convert grib files with multiple variables to Total Accumulated Precipitation GeoTIFFs.
Dependencies: datetime, os, shutil, numpy, rasterio
Script to convert grib files with multiple variables to Total Accumulated Precipitation GeoTIFFs . Dependencies : datetime os shutil numpy rasterio
def wrfpr_tiffs(threddspath, wrksppath, timestamp, region):
    logging.info('\nStarting to process the WRF-PR gribs into GeoTIFFs')
    tiffs = os.path.join(wrksppath, region, 'wrfpr_GeoTIFFs')
    gribs = os.path.join(threddspath, region, 'wrfpr', timestamp, 'gribs')
    netcdfs = os.path.join(threddspath, region, 'wrfpr', timestamp, 'netcdfs')
    if not os.path.exists(gribs):
        logging.info('There is no gribs folder, you must have already run this step. Skipping conversions')
        return
    else:
        shutil.rmtree(netcdfs)
        os.mkdir(netcdfs)
        os.chmod(netcdfs, 0o777)
        shutil.rmtree(tiffs)
        os.mkdir(tiffs)
        os.chmod(tiffs, 0o777)
    files = os.listdir(gribs)
    files = [grib for grib in files if grib.endswith('.grb')]
    files.sort()
    path = os.path.join(gribs, files[0])
    raster_dim = rasterio.open(path)
    width = raster_dim.width
    height = raster_dim.height
    lon_min = raster_dim.bounds.left
    lon_max = raster_dim.bounds.right
    lat_min = raster_dim.bounds.bottom
    lat_max = raster_dim.bounds.top
    geotransform = rasterio.transform.from_bounds(lon_min, lat_min, lon_max, lat_max, width, height)
    for i, j in zip(files, range(len(files))):
        logging.info('working on file ' + i)
        if j == 0:
            path = os.path.join(gribs, i)
            src = rasterio.open(path)
            file_array = src.read(1)
        else:
            cum_file = os.path.join(gribs, i)
            cum_src = rasterio.open(cum_file)
            cum_array = cum_src.read(1)
            past_file = os.path.join(gribs, files[j-1])
            past_src = rasterio.open(past_file)
            past_array = past_src.read(1)
            file_array = cum_array - past_array
        logging.info('opening grib file ' + path)
        obj = xarray.open_dataset(path, engine='cfgrib', backend_kwargs={'filter_by_keys': {'typeOfLevel': 'surface'}})
        logging.info('converting it to a netcdf')
        ncname = i.replace('.grb', '.nc')
        logging.info('saving it to the path ' + path)
        ncpath = os.path.join(netcdfs, ncname)
        obj.to_netcdf(ncpath, mode='w')
        logging.info('converted')
        logging.info('writing the correct values to the tp array')
        nc = netCDF4.Dataset(ncpath, 'a')
        nc['tp'][:] = file_array
        nc.close()
        logging.info('created a netcdf')
        tif_filename = i.replace('grb', 'tif')
        tif_filepath = os.path.join(tiffs, tif_filename)
        with rasterio.open(
                tif_filepath,
                'w',
                driver='GTiff',
                height=file_array.shape[0],
                width=file_array.shape[1],
                count=1,
                dtype=file_array.dtype,
                nodata=numpy.nan,
                crs='+proj=latlong',
                transform=geotransform,
        ) as dst:
            dst.write(file_array, 1)
        logging.info('wrote it to a GeoTIFF\n')
    shutil.rmtree(gribs)
    return
def wrfpr_tiffs ( threddspath , wrksppath , timestamp , region ) : logging . info ( '\nStarting to process the WRF-PR gribs into GeoTIFFs' ) tiffs = os . path . join ( wrksppath , region , 'wrfpr_GeoTIFFs' ) gribs = os . path . join ( threddspath , region , 'wrfpr' , timestamp , 'gribs' ) netcdfs = os . path . join ( threddspath , region , 'wrfpr' , timestamp , 'netcdfs' ) if not os . path . exists ( gribs ) : logging . info ( 'There is no gribs folder, you must have already run this step. Skipping conversions' ) return else : shutil . rmtree ( netcdfs ) os . mkdir ( netcdfs ) os . chmod ( netcdfs , 0o777 ) shutil . rmtree ( tiffs ) os . mkdir ( tiffs ) os . chmod ( tiffs , 0o777 ) files = os . listdir ( gribs ) files = [ grib for grib in files if grib . endswith ( '.grb' ) ] files . sort ( ) path = os . path . join ( gribs , files [ 0 ] ) raster_dim = rasterio . open ( path ) width = raster_dim . width height = raster_dim . height lon_min = raster_dim . bounds . left lon_max = raster_dim . bounds . right lat_min = raster_dim . bounds . bottom lat_max = raster_dim . bounds . top geotransform = rasterio . transform . from_bounds ( lon_min , lat_min , lon_max , lat_max , width , height ) for i , j in zip ( files , range ( len ( files ) ) ) : logging . info ( 'working on file ' + i ) if j == 0 : path = os . path . join ( gribs , i ) src = rasterio . open ( path ) file_array = src . read ( 1 ) else : cum_file = os . path . join ( gribs , i ) cum_src = rasterio . open ( cum_file ) cum_array = cum_src . read ( 1 ) past_file = os . path . join ( gribs , files [ j - 1 ] ) past_src = rasterio . open ( past_file ) past_array = past_src . read ( 1 ) file_array = cum_array - past_array logging . info ( 'opening grib file ' + path ) obj = xarray . open_dataset ( path , engine = 'cfgrib' , backend_kwargs = { 'filter_by_keys' : { 'typeOfLevel' : 'surface' } } ) logging . info ( 'converting it to a netcdf' ) ncname = i . replace ( '.grb' , '.nc' ) logging . info ( 'saving it to the path ' + path ) ncpath = os . path . join ( netcdfs , ncname ) obj . to_netcdf ( ncpath , mode = 'w' ) logging . info ( 'converted' ) logging . info ( 'writing the correct values to the tp array' ) nc = netCDF4 . Dataset ( ncpath , 'a' ) nc [ 'tp' ] [ : ] = file_array nc . close ( ) logging . info ( 'created a netcdf' ) tif_filename = i . replace ( 'grb' , 'tif' ) tif_filepath = os . path . join ( tiffs , tif_filename ) with rasterio . open ( tif_filepath , 'w' , driver = 'GTiff' , height = file_array . shape [ 0 ] , width = file_array . shape [ 1 ] , count = 1 , dtype = file_array . dtype , nodata = numpy . nan , crs = '+proj=latlong' , transform = geotransform , ) as dst : dst . write ( file_array , 1 ) logging . info ( 'wrote it to a GeoTIFF\n' ) shutil . rmtree ( gribs ) return
Script to convert grib files with multiple variables to Total Accumulated Precipitation GeoTIFFs.
Script to convert grib files with multiple variables to Total Accumulated Precipitation GeoTIFFs .
"""
    Script to convert grib files with multiple variables to Total Accumulated Precipitation GeoTIFFs.
    Dependencies: datetime, os, shutil, numpy, rasterio
    """ # declare the environment # if you already have wrf-pr netcdfs in the netcdfs folder, quit the function # otherwise, remove anything in the folder before starting (in case there was a partial conversion) # create a list of all the files of type "grb" and convert to a list of their file paths # Read raster dimensions only once to apply to all rasters # Geotransform for each 24-hr raster (east, south, west, north, width, height) # Create 1-hr GeoTIFFs and NetCDFs # using the last grib file for the day (path) convert it to a netcdf and set the variable to file_array # Specify the GeoTIFF filepath # Save the 24-hr raster # # clear the gribs folder now that we're done with this
threddspath
wrksppath
timestamp
region