obs4MIPs/obs4MIPs_process.py
def process( rc ):
    '''
    Convert netcdf/matlab/grads files into CMIP5 format.
    '''
    pdb.set_trace()
    # ----------------------------
    #  Loop yearly on file list.  
    # ----------------------------
    file_template = rc['file_template'].split(",");
    if( len(file_template) == 2 ):
        template_parameter = file_template[1]
        rc['file_template'] = file_template[0]
    else:
        template_parameter = 'years'

    for year in rc[template_parameter].split(","):
        if(year == ''):
            files= os.popen( "ls " + rc['file_template'] ).readlines()
        else:
            # ------------------------------------------------
            # Use string formating for path with same argument 
            # ------------------------------------------------
            try:
                tmplFile = rc['file_template'] % (year)
            except:
                tmplFile = rc['file_template'].format(year)
            if( not os.path.isfile( tmplFile) ) :
                print("**** Warning {} not found\n".format(tmplFile))
                continue
            files= os.popen( "ls " + tmplFile).readlines()

        if( files == [] ):
            print("No file found: Check your resource file")
            return -1
        # ------------------------------------------------
        # Get the right handler to manage this file format
        # ------------------------------------------------
        Handler = factory.HandlerFormats(files[0].strip())

        # -----------------------------------
        # Take care of cmor initialization.
        # -----------------------------------
        cmor.setup(inpath=rc['inpath'],
                   netcdf_file_action = cmor.CMOR_REPLACE)
      
        cmor.dataset(experiment_id = rc['experiment_id'], 
                     institution   = rc['institution'  ],
                     calendar      = rc['calendar'     ],
                     institute_id  = rc['institute_id' ],
                     model_id      = rc['model_id'     ],
                     source        = rc['source'       ],
                     contact       = rc['contact'      ],
                     references    = rc['references'   ])
      
	
        # -----------------------------------------
        # add extra Global Attributes for obs4MIPs.
        # -----------------------------------------
        cmor.set_cur_dataset_attribute( 'instrument',     rc['instrument'    ])
        cmor.set_cur_dataset_attribute( 'mip_specs',      rc['mip_specs'     ])
        cmor.set_cur_dataset_attribute( 'data_structure', rc['data_structure'])
        cmor.set_cur_dataset_attribute( 'source_type',    rc['source_type'   ])
        cmor.set_cur_dataset_attribute( 'source_id',      rc['source_id'     ])
        cmor.set_cur_dataset_attribute( 'realm',          rc['realm'         ])
        cmor.set_cur_dataset_attribute( 'obs_project',    rc['obs_project'   ])
        cmor.set_cur_dataset_attribute( 'processing_version',
                                        rc['processing_version'] )
        cmor.set_cur_dataset_attribute( 'processing_level',
                                        rc['processing_level'] )

        cmor.load_table(rc['table'])

        # ---------------------------------------------------------------------
        # We loop on each file found, a new cmor file will be create on each
        # iteration.  If you want to aggregate, you need to use Grads ctl file
        # or NeCDF list of file.
        # ---------------------------------------------------------------------
        for file in files:
	
            fnm=file.strip()    # Get rid of \n
            aVariable  = eval(rc['original_var'])
            nbVariable = len(aVariable)

            # -----------------------------------------------------
            # ECMWF needs synoptic time 00z and 12z in he filename.
            # We extract it from the first file in the list.
            # -----------------------------------------------------
            if( rc['source_fn'] == 'SYNOPTIC' ):
                index = fnm.find('z.')
                rc['SYNOPTIC'] = fnm[index-2:index]

            # -----------------------
            # Work on all variables
            # -------------------------
            for j in arange(nbVariable):
                # ----------------------------------------------------
                # Fetch the variable directly or excute equation.
                # ----------------------------------------------------
                try:
                    variable=aVariable[j]
                    Handler.open(fnm, variable=variable)
                    rc['cvrt_original_var']   = aVariable[j]
                    print("Working on variable {} ".format(variable))
                except:
                    if( aVariable[j] != 'equation' ) :
                        print("Variable {} can't open".format(variable))
                        continue
                    else:
                        print("Executing {} ".format(eval(rc['equation'])[j]))
                
#                pdb.set_trace()
                rc['cvrt_original_units'] = eval(rc['original_units'])[j]
                rc['cvrt_cmor_var']       = eval(rc['cmor_var'])[j]
                rc['cvrt_equation']       = eval(rc['equation'])[j]
                rc['cvrt_level']          = eval(rc['level'])[j]
                
                data=Handler.getData()

                # ----------------------------------------------------------
                # Evaluate equation if needed. Usually used to change units
                # ----------------------------------------------------------
                if( rc['cvrt_equation'][0] == '@' ):
                    fncName = rc['cvrt_equation'][1:]
                    fnc = import_equation( "equations.%s" %  fncName )
                    data[:]= fnc(Handler)
                else:
                    data[:]=eval(rc['cvrt_equation'])
         
                # -------------------------------------------------------------
                # Save filled value in case data type is changed in createAxes
                # -------------------------------------------------------------
                fill_value = data.fill_value

                #  ---------------------------------------------
                #  Extract latitude/longitude
                #  ---------------------------------------------
                lonvals=Handler.getLongitude()
                latvals=Handler.getLatitude()
                # ---------------------
                # Create cmor time axis
                # ----------------------
                (rel_time, rel_time_bnds) = createTime(Handler, rc)

                # ---------------------------------------------------
                # Create cmor axes and add an axis to data if needed
                # ---------------------------------------------------
                (axes, data) = createAxes( rc, latvals, lonvals, data )
                                
                axis_ids = list()
                for axis in axes:
                    axis_id = cmor.axis(**axis)
                    axis_ids.append(axis_id)
            
                # ----------------------------------------------------------
                # Create cmor variable
                # Note: since this is in the loop, a new cmor file will be
                # create for each cmor write command.
                # ----------------------------------------------------------
                varid = cmor.variable(table_entry   = rc['cvrt_cmor_var'],
                                      axis_ids      = axis_ids,
                                      history       = '',
                                      missing_value = fill_value,
                                      original_name = rc['cvrt_original_var'],
                                      units         = rc['cvrt_original_units']
                                      )
                # -------------------------------
                # Write data for this time frame.
                # -------------------------------
                cmor.write(varid,data,\
                           time_vals=rel_time,time_bnds=rel_time_bnds)
                cmor.close(varid,file_name=True)

                # ---------------------------------------
                # Rename cmor files according to project.
                # ---------------------------------------
                if( movefiles(rc) ):
                    return -2 

        cmor.close()
    return 0

    Convert netcdf/matlab/grads files into CMIP5 format.
    
Convert netcdf/matlab/grads files into CMIP5 format.
Convert netcdf / matlab / grads files into CMIP5 format .
def process( rc ):
    pdb.set_trace()
    file_template = rc['file_template'].split(",");
    if( len(file_template) == 2 ):
        template_parameter = file_template[1]
        rc['file_template'] = file_template[0]
    else:
        template_parameter = 'years'
    for year in rc[template_parameter].split(","):
        if(year == ''):
            files= os.popen( "ls " + rc['file_template'] ).readlines()
        else:
            --------------------
            --------------------
            try:
                tmplFile = rc['file_template'] % (year)
            except:
                tmplFile = rc['file_template'].format(year)
            if( not os.path.isfile( tmplFile) ) :
                print("**** Warning {} not found\n".format(tmplFile))
                continue
            files= os.popen( "ls " + tmplFile).readlines()
        if( files == [] ):
            print("No file found: Check your resource file")
            return -1
        --------------------
        --------------------
        Handler = factory.HandlerFormats(files[0].strip())
        -------
        -------
        cmor.setup(inpath=rc['inpath'],
                   netcdf_file_action = cmor.CMOR_REPLACE)
        cmor.dataset(experiment_id = rc['experiment_id'], 
                     institution   = rc['institution'  ],
                     calendar      = rc['calendar'     ],
                     institute_id  = rc['institute_id' ],
                     model_id      = rc['model_id'     ],
                     source        = rc['source'       ],
                     contact       = rc['contact'      ],
                     references    = rc['references'   ])
        -------------
        -------------
        cmor.set_cur_dataset_attribute( 'instrument',     rc['instrument'    ])
        cmor.set_cur_dataset_attribute( 'mip_specs',      rc['mip_specs'     ])
        cmor.set_cur_dataset_attribute( 'data_structure', rc['data_structure'])
        cmor.set_cur_dataset_attribute( 'source_type',    rc['source_type'   ])
        cmor.set_cur_dataset_attribute( 'source_id',      rc['source_id'     ])
        cmor.set_cur_dataset_attribute( 'realm',          rc['realm'         ])
        cmor.set_cur_dataset_attribute( 'obs_project',    rc['obs_project'   ])
        cmor.set_cur_dataset_attribute( 'processing_version',
                                        rc['processing_version'] )
        cmor.set_cur_dataset_attribute( 'processing_level',
                                        rc['processing_level'] )
        cmor.load_table(rc['table'])
        -----------------------------------------
        -----------------------------------------
        for file in files:
            fnm=file.strip()    
            aVariable  = eval(rc['original_var'])
            nbVariable = len(aVariable)
            -------------------------
            -------------------------
            if( rc['source_fn'] == 'SYNOPTIC' ):
                index = fnm.find('z.')
                rc['SYNOPTIC'] = fnm[index-2:index]
            --
            for j in arange(nbVariable):
                ------------------------
                ------------------------
                try:
                    variable=aVariable[j]
                    Handler.open(fnm, variable=variable)
                    rc['cvrt_original_var']   = aVariable[j]
                    print("Working on variable {} ".format(variable))
                except:
                    if( aVariable[j] != 'equation' ) :
                        print("Variable {} can't open".format(variable))
                        continue
                    else:
                        print("Executing {} ".format(eval(rc['equation'])[j]))
                rc['cvrt_original_units'] = eval(rc['original_units'])[j]
                rc['cvrt_cmor_var']       = eval(rc['cmor_var'])[j]
                rc['cvrt_equation']       = eval(rc['equation'])[j]
                rc['cvrt_level']          = eval(rc['level'])[j]
                data=Handler.getData()
                ------------------------------
                ------------------------------
                if( rc['cvrt_equation'][0] == '@' ):
                    fncName = rc['cvrt_equation'][1:]
                    fnc = import_equation( "equations.%s" %  fncName )
                    data[:]= fnc(Handler)
                else:
                    data[:]=eval(rc['cvrt_equation'])
                ---------------------------------
                ---------------------------------
                fill_value = data.fill_value
                lonvals=Handler.getLongitude()
                latvals=Handler.getLatitude()
                -
                (rel_time, rel_time_bnds) = createTime(Handler, rc)
                -----------------------
                -----------------------
                (axes, data) = createAxes( rc, latvals, lonvals, data )
                axis_ids = list()
                for axis in axes:
                    axis_id = cmor.axis(**axis)
                    axis_ids.append(axis_id)
                ------------------------------
                ------------------------------
                varid = cmor.variable(table_entry   = rc['cvrt_cmor_var'],
                                      axis_ids      = axis_ids,
                                      history       = '',
                                      missing_value = fill_value,
                                      original_name = rc['cvrt_original_var'],
                                      units         = rc['cvrt_original_units']
                                      )
                ---
                ---
                cmor.write(varid,data,\
                           time_vals=rel_time,time_bnds=rel_time_bnds)
                cmor.close(varid,file_name=True)
                -----------
                -----------
                if( movefiles(rc) ):
                    return -2 
        cmor.close()
    return 0
def process ( rc ) : pdb . set_trace ( ) file_template = rc [ 'file_template' ] . split ( "," ) ; if ( len ( file_template ) == 2 ) : template_parameter = file_template [ 1 ] rc [ 'file_template' ] = file_template [ 0 ] else : template_parameter = 'years' for year in rc [ template_parameter ] . split ( "," ) : if ( year == '' ) : files = os . popen ( "ls " + rc [ 'file_template' ] ) . readlines ( ) else : try : tmplFile = rc [ 'file_template' ] % ( year ) except : tmplFile = rc [ 'file_template' ] . format ( year ) if ( not os . path . isfile ( tmplFile ) ) : print ( "**** Warning {} not found\n" . format ( tmplFile ) ) continue files = os . popen ( "ls " + tmplFile ) . readlines ( ) if ( files == [ ] ) : print ( "No file found: Check your resource file" ) return - 1 Handler = factory . HandlerFormats ( files [ 0 ] . strip ( ) ) cmor . setup ( inpath = rc [ 'inpath' ] , netcdf_file_action = cmor . CMOR_REPLACE ) cmor . dataset ( experiment_id = rc [ 'experiment_id' ] , institution = rc [ 'institution' ] , calendar = rc [ 'calendar' ] , institute_id = rc [ 'institute_id' ] , model_id = rc [ 'model_id' ] , source = rc [ 'source' ] , contact = rc [ 'contact' ] , references = rc [ 'references' ] ) cmor . set_cur_dataset_attribute ( 'instrument' , rc [ 'instrument' ] ) cmor . set_cur_dataset_attribute ( 'mip_specs' , rc [ 'mip_specs' ] ) cmor . set_cur_dataset_attribute ( 'data_structure' , rc [ 'data_structure' ] ) cmor . set_cur_dataset_attribute ( 'source_type' , rc [ 'source_type' ] ) cmor . set_cur_dataset_attribute ( 'source_id' , rc [ 'source_id' ] ) cmor . set_cur_dataset_attribute ( 'realm' , rc [ 'realm' ] ) cmor . set_cur_dataset_attribute ( 'obs_project' , rc [ 'obs_project' ] ) cmor . set_cur_dataset_attribute ( 'processing_version' , rc [ 'processing_version' ] ) cmor . set_cur_dataset_attribute ( 'processing_level' , rc [ 'processing_level' ] ) cmor . load_table ( rc [ 'table' ] ) for file in files : fnm = file . strip ( ) aVariable = eval ( rc [ 'original_var' ] ) nbVariable = len ( aVariable ) if ( rc [ 'source_fn' ] == 'SYNOPTIC' ) : index = fnm . find ( 'z.' ) rc [ 'SYNOPTIC' ] = fnm [ index - 2 : index ] for j in arange ( nbVariable ) : try : variable = aVariable [ j ] Handler . open ( fnm , variable = variable ) rc [ 'cvrt_original_var' ] = aVariable [ j ] print ( "Working on variable {} " . format ( variable ) ) except : if ( aVariable [ j ] != 'equation' ) : print ( "Variable {} can't open" . format ( variable ) ) continue else : print ( "Executing {} " . format ( eval ( rc [ 'equation' ] ) [ j ] ) ) rc [ 'cvrt_original_units' ] = eval ( rc [ 'original_units' ] ) [ j ] rc [ 'cvrt_cmor_var' ] = eval ( rc [ 'cmor_var' ] ) [ j ] rc [ 'cvrt_equation' ] = eval ( rc [ 'equation' ] ) [ j ] rc [ 'cvrt_level' ] = eval ( rc [ 'level' ] ) [ j ] data = Handler . getData ( ) if ( rc [ 'cvrt_equation' ] [ 0 ] == '@' ) : fncName = rc [ 'cvrt_equation' ] [ 1 : ] fnc = import_equation ( "equations.%s" % fncName ) data [ : ] = fnc ( Handler ) else : data [ : ] = eval ( rc [ 'cvrt_equation' ] ) fill_value = data . fill_value lonvals = Handler . getLongitude ( ) latvals = Handler . getLatitude ( ) ( rel_time , rel_time_bnds ) = createTime ( Handler , rc ) ( axes , data ) = createAxes ( rc , latvals , lonvals , data ) axis_ids = list ( ) for axis in axes : axis_id = cmor . axis ( ** axis ) axis_ids . append ( axis_id ) varid = cmor . variable ( table_entry = rc [ 'cvrt_cmor_var' ] , axis_ids = axis_ids , history = '' , missing_value = fill_value , original_name = rc [ 'cvrt_original_var' ] , units = rc [ 'cvrt_original_units' ] ) cmor . write ( varid , data , time_vals = rel_time , time_bnds = rel_time_bnds ) cmor . close ( varid , file_name = True ) if ( movefiles ( rc ) ) : return - 2 cmor . close ( ) return 0
Convert netcdf/matlab/grads files into CMIP5 format.
Convert netcdf / matlab / grads files into CMIP5 format .
'''
    Convert netcdf/matlab/grads files into CMIP5 format.
    ''' # ---------------------------- #  Loop yearly on file list.   # ---------------------------- # ------------------------------------------------ # Use string formating for path with same argument  # ------------------------------------------------ # ------------------------------------------------ # Get the right handler to manage this file format # ------------------------------------------------ # ----------------------------------- # Take care of cmor initialization. # ----------------------------------- # ----------------------------------------- # add extra Global Attributes for obs4MIPs. # ----------------------------------------- # --------------------------------------------------------------------- # We loop on each file found, a new cmor file will be create on each # iteration.  If you want to aggregate, you need to use Grads ctl file # or NeCDF list of file. # --------------------------------------------------------------------- # Get rid of \n # ----------------------------------------------------- # ECMWF needs synoptic time 00z and 12z in he filename. # We extract it from the first file in the list. # ----------------------------------------------------- # ----------------------- # Work on all variables # ------------------------- # ---------------------------------------------------- # Fetch the variable directly or excute equation. # ---------------------------------------------------- #                pdb.set_trace() # ---------------------------------------------------------- # Evaluate equation if needed. Usually used to change units # ---------------------------------------------------------- # ------------------------------------------------------------- # Save filled value in case data type is changed in createAxes # ------------------------------------------------------------- #  --------------------------------------------- #  Extract latitude/longitude #  --------------------------------------------- # --------------------- # Create cmor time axis # ---------------------- # --------------------------------------------------- # Create cmor axes and add an axis to data if needed # --------------------------------------------------- # ---------------------------------------------------------- # Create cmor variable # Note: since this is in the loop, a new cmor file will be # create for each cmor write command. # ---------------------------------------------------------- # ------------------------------- # Write data for this time frame. # ------------------------------- # --------------------------------------- # Rename cmor files according to project. # ---------------------------------------
rc