data_workflow/gfsworkflow.py
def run_gfs_workflow(threddspath):
    """
    The controller for running the workflow to download and process data
    """
    wrksppath = os.path.join(
        os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'tethysapp', 'ffgs', 'workspaces', 'app_workspace')

    # enable logging to track the progress of the workflow and for debugging
    logpath = os.path.join(threddspath, 'workflow.log')
    logging.basicConfig(filename=logpath, filemode='w', level=logging.INFO, format='%(message)s')
    logging.info('Workflow initiated on ' + datetime.datetime.utcnow().strftime("%D at %R"))

    # start the workflow by setting the environment
    timestamp, redundant = setenvironment(threddspath, wrksppath)
    model = 'gfs'

    # if this has already been done for the most recent forecast, abort the workflow
    if redundant:
        logging.info('\nWorkflow aborted on ' + datetime.datetime.utcnow().strftime("%D at %R"))
        return 'Workflow Aborted- already run for most recent data'

    # run the workflow for each region, for each model in that region
    for region in FFGS_REGIONS:
        logging.info('\nBeginning to process ' + region[1] + ' on ' + datetime.datetime.utcnow().strftime("%D at %R"))
        # download each forecast model, convert them to netcdfs and tiffs
        succeeded = download_gfs(threddspath, timestamp, region[1], model)
        if not succeeded:
            return 'Workflow Aborted- Downloading Errors Occurred'
        gfs_tiffs(threddspath, wrksppath, timestamp, region[1], model)
        resample(wrksppath, region[1], model)
        # the geoprocessing functions
        zonal_statistics(wrksppath, timestamp, region[1], model)
        nc_georeference(threddspath, timestamp, region[1], model)
        # generate color scales and ncml aggregation files
        new_ncml(threddspath, timestamp, region[1], model)
        new_colorscales(wrksppath, region[1], model)
        # cleanup the workspace by removing old files
        cleanup(threddspath, timestamp, region[1], model)

    logging.info('\nAll regions finished- writing the timestamp used on this run to a txt file')
    with open(os.path.join(threddspath, 'gfs_timestamp.txt'), 'w') as file:
        file.write(timestamp)

    logging.info('\n\nGFS Workflow completed successfully on ' + datetime.datetime.utcnow().strftime("%D at %R"))
    logging.info('If you have configured other models, they will begin processing now.\n\n\n')

    return 'GFS Workflow Completed- Normal Finish'

    The controller for running the workflow to download and process data
    
The controller for running the workflow to download and process data
The controller for running the workflow to download and process data
def run_gfs_workflow(threddspath):
    wrksppath = os.path.join(
        os.path.dirname(os.path.dirname(os.path.abspath(__file__))), 'tethysapp', 'ffgs', 'workspaces', 'app_workspace')
    logpath = os.path.join(threddspath, 'workflow.log')
    logging.basicConfig(filename=logpath, filemode='w', level=logging.INFO, format='%(message)s')
    logging.info('Workflow initiated on ' + datetime.datetime.utcnow().strftime("%D at %R"))
    timestamp, redundant = setenvironment(threddspath, wrksppath)
    model = 'gfs'
    if redundant:
        logging.info('\nWorkflow aborted on ' + datetime.datetime.utcnow().strftime("%D at %R"))
        return 'Workflow Aborted- already run for most recent data'
    for region in FFGS_REGIONS:
        logging.info('\nBeginning to process ' + region[1] + ' on ' + datetime.datetime.utcnow().strftime("%D at %R"))
        succeeded = download_gfs(threddspath, timestamp, region[1], model)
        if not succeeded:
            return 'Workflow Aborted- Downloading Errors Occurred'
        gfs_tiffs(threddspath, wrksppath, timestamp, region[1], model)
        resample(wrksppath, region[1], model)
        zonal_statistics(wrksppath, timestamp, region[1], model)
        nc_georeference(threddspath, timestamp, region[1], model)
        new_ncml(threddspath, timestamp, region[1], model)
        new_colorscales(wrksppath, region[1], model)
        cleanup(threddspath, timestamp, region[1], model)
    logging.info('\nAll regions finished- writing the timestamp used on this run to a txt file')
    with open(os.path.join(threddspath, 'gfs_timestamp.txt'), 'w') as file:
        file.write(timestamp)
    logging.info('\n\nGFS Workflow completed successfully on ' + datetime.datetime.utcnow().strftime("%D at %R"))
    logging.info('If you have configured other models, they will begin processing now.\n\n\n')
    return 'GFS Workflow Completed- Normal Finish'
def run_gfs_workflow ( threddspath ) : wrksppath = os . path . join ( os . path . dirname ( os . path . dirname ( os . path . abspath ( __file__ ) ) ) , 'tethysapp' , 'ffgs' , 'workspaces' , 'app_workspace' ) logpath = os . path . join ( threddspath , 'workflow.log' ) logging . basicConfig ( filename = logpath , filemode = 'w' , level = logging . INFO , format = '%(message)s' ) logging . info ( 'Workflow initiated on ' + datetime . datetime . utcnow ( ) . strftime ( "%D at %R" ) ) timestamp , redundant = setenvironment ( threddspath , wrksppath ) model = 'gfs' if redundant : logging . info ( '\nWorkflow aborted on ' + datetime . datetime . utcnow ( ) . strftime ( "%D at %R" ) ) return 'Workflow Aborted- already run for most recent data' for region in FFGS_REGIONS : logging . info ( '\nBeginning to process ' + region [ 1 ] + ' on ' + datetime . datetime . utcnow ( ) . strftime ( "%D at %R" ) ) succeeded = download_gfs ( threddspath , timestamp , region [ 1 ] , model ) if not succeeded : return 'Workflow Aborted- Downloading Errors Occurred' gfs_tiffs ( threddspath , wrksppath , timestamp , region [ 1 ] , model ) resample ( wrksppath , region [ 1 ] , model ) zonal_statistics ( wrksppath , timestamp , region [ 1 ] , model ) nc_georeference ( threddspath , timestamp , region [ 1 ] , model ) new_ncml ( threddspath , timestamp , region [ 1 ] , model ) new_colorscales ( wrksppath , region [ 1 ] , model ) cleanup ( threddspath , timestamp , region [ 1 ] , model ) logging . info ( '\nAll regions finished- writing the timestamp used on this run to a txt file' ) with open ( os . path . join ( threddspath , 'gfs_timestamp.txt' ) , 'w' ) as file : file . write ( timestamp ) logging . info ( '\n\nGFS Workflow completed successfully on ' + datetime . datetime . utcnow ( ) . strftime ( "%D at %R" ) ) logging . info ( 'If you have configured other models, they will begin processing now.\n\n\n' ) return 'GFS Workflow Completed- Normal Finish'
The controller for running the workflow to download and process data
The controller for running the workflow to download and process data
"""
    The controller for running the workflow to download and process data
    """ # enable logging to track the progress of the workflow and for debugging # start the workflow by setting the environment # if this has already been done for the most recent forecast, abort the workflow # run the workflow for each region, for each model in that region # download each forecast model, convert them to netcdfs and tiffs # the geoprocessing functions # generate color scales and ncml aggregation files # cleanup the workspace by removing old files
threddspath