data_workflow/wrfprworkflow.py
def download_wrfpr(threddspath, timestamp, region):
    """
    Script to download WRF-PuertoRico Grib Files.
    Dependencies: datetime, os, requests, shutil
    """
    logging.info('\nStarting WRF-PR Grib Downloads')
    # set filepaths
    gribsdir = os.path.join(threddspath, region, 'wrfpr', timestamp, 'gribs')

    # if you already have a folder with data for this timestep, quit this function (you dont need to download it)
    if not os.path.exists(gribsdir):
        logging.info('There is no download folder, you must have already processed them. Skipping download stage.')
        return True
    elif len(os.listdir(gribsdir)) >= 48:
        logging.info('There are already 48 forecast steps in here. Dont need to download them')
        return True
    # otherwise, remove anything in the folder before starting (in case there was a partial download)
    else:
        shutil.rmtree(gribsdir)
        os.mkdir(gribsdir)
        os.chmod(gribsdir, 0o777)

    # # get the parts of the timestamp to put into the url
    time = datetime.datetime.strptime(timestamp, "%Y%m%d%H").strftime("%H")
    fc_date = datetime.datetime.strptime(timestamp, "%Y%m%d%H").strftime("%Y%m%d")

    # This is the List of forecast timesteps for 5 days (6-hr increments). download them all
    fc_steps = ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12',
                '13', '14', '15', '16', '17', '18', '19', '20', '21', '22', '23', '24',
                '25', '26', '27', '28', '29', '30', '31', '32', '33', '34', '35', '36',
                '37', '38', '39', '40', '41', '42', '43', '44', '45', '46', '47', '48']

    # this is where the actual downloads happen. set the url, filepath, then download
    for step in fc_steps:
        url = 'https://nomads.ncep.noaa.gov/cgi-bin/filter_hirespr.pl?file=hiresw.t' + time + 'z.arw_5km.f' + step + \
              '.pr.grib2&lev_surface=on&var_APCP=on&leftlon=0&rightlon=360&toplat=90&bottomlat=-90&dir=%2Fhiresw.' + \
              fc_date

        fc_timestamp = datetime.datetime.strptime(timestamp, "%Y%m%d%H")
        file_timestep = fc_timestamp + datetime.timedelta(hours=int(step))
        filename_timestep = datetime.datetime.strftime(file_timestep, "%Y%m%d%H")

        filename = filename_timestep + '.grb'
        logging.info('downloading the file ' + filename)
        filepath = os.path.join(gribsdir, filename)
        try:
            with requests.get(url, stream=True) as r:
                r.raise_for_status()
                with open(filepath, 'wb') as f:
                    for chunk in r.iter_content(chunk_size=8192):
                        if chunk:  # filter out keep-alive new chunks
                            f.write(chunk)
        except requests.HTTPError as e:
            errorcode = e.response.status_code
            logging.info('\nHTTPError ' + str(errorcode) + ' downloading ' + filename + ' from\n' + url)
            if errorcode == 404:
                logging.info('The file was not found on the server, trying an older forecast time')
            elif errorcode == 500:
                logging.info('Probably a problem with the URL. Check the log and try the link')
            return False
    logging.info('Finished Downloads')
    return True

    Script to download WRF-PuertoRico Grib Files.
    Dependencies: datetime, os, requests, shutil
    
Script to download WRF-PuertoRico Grib Files.
Dependencies: datetime, os, requests, shutil
Script to download WRF - PuertoRico Grib Files . Dependencies : datetime os requests shutil
def download_wrfpr(threddspath, timestamp, region):
    logging.info('\nStarting WRF-PR Grib Downloads')
    gribsdir = os.path.join(threddspath, region, 'wrfpr', timestamp, 'gribs')
    if not os.path.exists(gribsdir):
        logging.info('There is no download folder, you must have already processed them. Skipping download stage.')
        return True
    elif len(os.listdir(gribsdir)) >= 48:
        logging.info('There are already 48 forecast steps in here. Dont need to download them')
        return True
    else:
        shutil.rmtree(gribsdir)
        os.mkdir(gribsdir)
        os.chmod(gribsdir, 0o777)
    time = datetime.datetime.strptime(timestamp, "%Y%m%d%H").strftime("%H")
    fc_date = datetime.datetime.strptime(timestamp, "%Y%m%d%H").strftime("%Y%m%d")
    fc_steps = ['01', '02', '03', '04', '05', '06', '07', '08', '09', '10', '11', '12',
                '13', '14', '15', '16', '17', '18', '19', '20', '21', '22', '23', '24',
                '25', '26', '27', '28', '29', '30', '31', '32', '33', '34', '35', '36',
                '37', '38', '39', '40', '41', '42', '43', '44', '45', '46', '47', '48']
    for step in fc_steps:
        url = 'https://nomads.ncep.noaa.gov/cgi-bin/filter_hirespr.pl?file=hiresw.t' + time + 'z.arw_5km.f' + step + \
              '.pr.grib2&lev_surface=on&var_APCP=on&leftlon=0&rightlon=360&toplat=90&bottomlat=-90&dir=%2Fhiresw.' + \
              fc_date
        fc_timestamp = datetime.datetime.strptime(timestamp, "%Y%m%d%H")
        file_timestep = fc_timestamp + datetime.timedelta(hours=int(step))
        filename_timestep = datetime.datetime.strftime(file_timestep, "%Y%m%d%H")
        filename = filename_timestep + '.grb'
        logging.info('downloading the file ' + filename)
        filepath = os.path.join(gribsdir, filename)
        try:
            with requests.get(url, stream=True) as r:
                r.raise_for_status()
                with open(filepath, 'wb') as f:
                    for chunk in r.iter_content(chunk_size=8192):
                        if chunk:  
                            f.write(chunk)
        except requests.HTTPError as e:
            errorcode = e.response.status_code
            logging.info('\nHTTPError ' + str(errorcode) + ' downloading ' + filename + ' from\n' + url)
            if errorcode == 404:
                logging.info('The file was not found on the server, trying an older forecast time')
            elif errorcode == 500:
                logging.info('Probably a problem with the URL. Check the log and try the link')
            return False
    logging.info('Finished Downloads')
    return True
def download_wrfpr ( threddspath , timestamp , region ) : logging . info ( '\nStarting WRF-PR Grib Downloads' ) gribsdir = os . path . join ( threddspath , region , 'wrfpr' , timestamp , 'gribs' ) if not os . path . exists ( gribsdir ) : logging . info ( 'There is no download folder, you must have already processed them. Skipping download stage.' ) return True elif len ( os . listdir ( gribsdir ) ) >= 48 : logging . info ( 'There are already 48 forecast steps in here. Dont need to download them' ) return True else : shutil . rmtree ( gribsdir ) os . mkdir ( gribsdir ) os . chmod ( gribsdir , 0o777 ) time = datetime . datetime . strptime ( timestamp , "%Y%m%d%H" ) . strftime ( "%H" ) fc_date = datetime . datetime . strptime ( timestamp , "%Y%m%d%H" ) . strftime ( "%Y%m%d" ) fc_steps = [ '01' , '02' , '03' , '04' , '05' , '06' , '07' , '08' , '09' , '10' , '11' , '12' , '13' , '14' , '15' , '16' , '17' , '18' , '19' , '20' , '21' , '22' , '23' , '24' , '25' , '26' , '27' , '28' , '29' , '30' , '31' , '32' , '33' , '34' , '35' , '36' , '37' , '38' , '39' , '40' , '41' , '42' , '43' , '44' , '45' , '46' , '47' , '48' ] for step in fc_steps : url = 'https://nomads.ncep.noaa.gov/cgi-bin/filter_hirespr.pl?file=hiresw.t' + time + 'z.arw_5km.f' + step + '.pr.grib2&lev_surface=on&var_APCP=on&leftlon=0&rightlon=360&toplat=90&bottomlat=-90&dir=%2Fhiresw.' + fc_date fc_timestamp = datetime . datetime . strptime ( timestamp , "%Y%m%d%H" ) file_timestep = fc_timestamp + datetime . timedelta ( hours = int ( step ) ) filename_timestep = datetime . datetime . strftime ( file_timestep , "%Y%m%d%H" ) filename = filename_timestep + '.grb' logging . info ( 'downloading the file ' + filename ) filepath = os . path . join ( gribsdir , filename ) try : with requests . get ( url , stream = True ) as r : r . raise_for_status ( ) with open ( filepath , 'wb' ) as f : for chunk in r . iter_content ( chunk_size = 8192 ) : if chunk : f . write ( chunk ) except requests . HTTPError as e : errorcode = e . response . status_code logging . info ( '\nHTTPError ' + str ( errorcode ) + ' downloading ' + filename + ' from\n' + url ) if errorcode == 404 : logging . info ( 'The file was not found on the server, trying an older forecast time' ) elif errorcode == 500 : logging . info ( 'Probably a problem with the URL. Check the log and try the link' ) return False logging . info ( 'Finished Downloads' ) return True
Script to download WRF-PuertoRico Grib Files.
Script to download WRF - PuertoRico Grib Files .
"""
    Script to download WRF-PuertoRico Grib Files.
    Dependencies: datetime, os, requests, shutil
    """ # set filepaths # if you already have a folder with data for this timestep, quit this function (you dont need to download it) # otherwise, remove anything in the folder before starting (in case there was a partial download) # # get the parts of the timestamp to put into the url # This is the List of forecast timesteps for 5 days (6-hr increments). download them all # this is where the actual downloads happen. set the url, filepath, then download # filter out keep-alive new chunks
threddspath
timestamp
region