text: """A set of classes running checks and providing utilities to support checks"""
  import string, re, os, sys, traceback, ctypes, collections, importlib
  
  def strmm3( mm ):
    return string.join( map( lambda x: '%s="%s" [correct: "%s"]' % x, mm ), '; ' )
  
  from xceptions import *
  
  re_dlr01 = re.compile( 'MERGED-DLR_1M-[0-9]{8}-fv0100.nc$' )
  
  class timeInt(object):
  
     vc = {'gregorian':0, 'standard':0, 'proleptic_gregorian':0, 'noleap':1, '365_day':1, 'all_leap':2, '366_day':2, '360_day':3, 'julian':0, 'none':None}
     mnmx = [ (365,366),(365,365),(366,366),(360,360) ]
     mmnmmx = [ (28,31),(28,31),(29,31),(30,30) ]
     def __init__(self,cal='proleptic_gregorian',dpymn=None, dpymx=None,dpmmn=None,dpmmx=None,tol=1.e-6):
       self.tol = tol
       if not self.vc.has_key(cal) or cal is None:
         assert dpymx is not None and dpymn is not None, 'If standard calendar is not use, dpymn and dpymx must be set'
         assert dpmmx is not None and dpmmn is not None, 'If standard calendar is not use, dpmmn and dpmmx must be set'
         self.dpymn = dpymn - tol
         self.dpymx = dpymx + tol
         self.dpmmn = dpmmn - tol
         self.dpmmx = dpmmx + tol
       else:
         self.dpymn = self.mnmx[self.vc[cal]][0] - tol
         self.dpymx = self.mnmx[self.vc[cal]][1] + tol
         self.dpmmn = self.mmnmmx[self.vc[cal]][0] - tol
         self.dpmmx = self.mmnmmx[self.vc[cal]][1] + tol
       self.map = { 'yr':'P1Y','monClim':'P1M','mon':'P1M','day':'P1D','6hr':'P6H','3hr':'P3H', '1hr':'P1H'}
       self.nd = { 'x':'y' }
  
     def setUnit(self,u):
       if u not in ['days','months','years']:
           print  'Time unit %s not supported' % u
           self.u = None
       else:
           self.u = u
  
     def chk(self,v,u,f):
        if not self.map.has_key(f):
           return (0,'No frequency check available for f = %s' % f )
        if u not in ['days']:
           return (0,'No frequency check available for units = %s' % u )
        x = self.map(f)
        if x == 'P1Y':
          return (v > self.dpymn) and (v < self.dpymx)
        elif x == 'P1M':
          return (v > self.dpmmn) and (v < self.dpmmx)
        elif x == 'P1D':
          return (v > 1.-self.tol) and (v < 1.+self.tol)
        elif x == 'P6H':
          return (v > 0.25-self.tol) and (v < 0.25+self.tol)
        elif x == 'P3H':
          return (v > 0.125-self.tol) and (v < 0.125+self.tol)
        
  
  class reportSection(object):
  
    def __init__(self,id,cls,parent=None, description=None):
      self.id = id
      self.cls = cls
      self.parent = parent
      self.description = description
      self.records = []
      self.subsections = []
      self.closed = False
      self.npass = 0
      self.fail = 0
      self.auditDone = True
  
    def addSubSection( self, id, cls, description=None):
      assert not self.closed, 'Attempt to add sub-section to closed report section'
      self.subsections.append( reportSection(id, cls, parent=self, description=description )  )
      self.auditDone = False
      return self.subsections[-1]
  
    def addRecord( self, id, cls, res, msg ):
      assert not self.closed, 'Attempt to add record to closed report section'
      self.records.append( (id, cls, res, msg) )
      self.auditDone = False
  
    def close(self):
      self.closed = True
  
    def reopen(self):
      self.closed = False
  
    def audit(self):
      if self.auditDone:
        return
      self.closed = True
      self.fail = 0
      self.npass = 0
      for ss in self.subsections:
        ss.audit()
        self.fail += ss.fail
        self.npass += ss.npass
  
      for r in self.records:
        if r[2]:
          self.npass += 1
        else:
          self.fail += 1
  
  class checkSeq(object):
    def __init__(self):
      pass
  
    def check(self,x):
      d = map( lambda i: x[i+1] - x[i], range(len(x)-1) )
      self.delt = sum(d)/len(d)
      self.dmx = max(d)
      self.dmn = min(d)
      return self.dmx - self.dmn < abs(self.delt)*1.e-4
  
  cs = checkSeq()
  
  class checkBase(object):
    """Base class for checks, containing a set of standard methods for managing operation of checks and logging of results"""
  
    def  __init__(self,cls="CORDEX",reportPass=True,parent=None,monitor=None):
      """Creat class instance: set defaults, link arguments to instance, create a range of compiled regular expressions"""
      self.cls = cls
      self.project = cls
      self.abortMessageCount = parent.abortMessageCount
      self.monitor = monitor
      self.re_isInt = re.compile( '[0-9]+' )
      self.errorCount = 0
      self.passCount = 0
      self.missingValue = 1.e20
      self.missingValue = ctypes.c_float(1.00000002004e+20).value
      from file_utils import ncLib
      if ncLib == 'netCDF4':
        import numpy
        self.missingValue = numpy.float32(self.missingValue)
      self.parent = parent
      self.reportPass=reportPass
      self.pcfg = parent.pcfg
  ################################
      self.requiredGlobalAttributes = self.pcfg.requiredGlobalAttributes
      self.controlledGlobalAttributes = self.pcfg.controlledGlobalAttributes
      self.globalAttributesInFn = self.pcfg.globalAttributesInFn
      self.requiredVarAttributes = self.pcfg.requiredVarAttributes
      self.drsMappings = self.pcfg.drsMappings
  #######################################
      self.checks = ()
      self.plugins = ()
      self.agents = ()
      self.messageCount = 0
  
  ## call init method defined in implementing class 
      self.init()
  
  ## call initPlugs method to initialise any plugins defined by init
      self.__initPlugs__()
  
      if not hasattr( self.parent, 'amapListDraft' ):
        self.parent.amapListDraft = []
  
    def isInt(self,x):
      """Check that a string is a representation of an integer"""
      return self.re_isInt.match( x ) is not None
  
    def logMessage(self, msg, error=False ):
      """Log messages and count messages"""
      self.messageCount += 1
      assert self.abortMessageCount < 0 or self.abortMessageCount > self.messageCount, 'Raising error [TESTX01], perhaps for testing'
      if self.parent is not None and self.parent.log is not None:
         if error:
           self.parent.log.error( msg )
         else:
           self.parent.log.info( msg )
      else:
         print msg
  
      doThis = True
      if self.appendLogfile[0] is not None and doThis:
        if self.monitor is not None:
           nofh0 = self.monitor.get_open_fds()
        xlog = self.c4i.getFileLog( self.appendLogfile[1], flf=self.appendLogfile[0] )
        if error:
           xlog.error( msg )
        else:
           xlog.info( msg )
        self.c4i.closeFileLog()
        if self.monitor is not None:
           nofh9 = self.monitor.get_open_fds()
           if nofh9 > nofh0:
             print 'Leaking file handles [1]: %s --- %s' % (nofh0, nofh9)
  
    def log_exception( self, msg):
      """Logging of exceptions -- putting trace information in log files"""
      if self.parent is not None and self.parent.log is not None:
          self.parent.log.error("Exception has occured" ,exc_info=1)
      else:
          traceback.print_exc(file=sys.stdout)
  
    def log_error( self, msg ):
      """Create an error log message and call logMessage; count errors;"""
      self.lastError = msg
      self.errorCount += 1
      self.logMessage( '%s.%s: FAILED:: %s' % (self.id,self.getCheckId(),msg), error=True )
  
    def log_pass( self ):
      """Create a pass log message and call logMessage; count passes;"""
      self.passCount = True
      if self.reportPass:
        self.logMessage(  '%s.%s: OK' % (self.id,self.getCheckId()) )
  
    def log_abort( self ):
      self.completed = False
      self.logMessage(   '%s.%s: ABORTED:: Errors too severe to complete further checks in this module' % (self.id,'xxx') )
      raise abortChecks
  
    def status(self):
      return '%s.%s' % (self.id,self.getCheckId())
  
    def getCheckId(self,full=True):
      if type( self.checkId ) == type( 'x' ):
        return self.checkId
      else:
        if full:
          return '%s: [%s]' % self.checkId
        else:
          return self.checkId[0]
  
    def test(self,res,msg,abort=False,part=False,appendLogfile=(None,None)):
      """Handle test results. 
        :param res: [True/False] result of test;
        :param msg: Message describing the test;
        :param abort: {optional} Set True if checks should be aborted when test fails;
        :param part: {optional} Set True if this is a component of a test (logging of pass suppressed);
        :param appendLogfile: {optional} Allows results to be appended to pre-existing log file;
      """
      self.appendLogfile = appendLogfile
      if res:
        if not part:
           self.log_pass()
      else:
        self.log_error(msg)
        if abort:
          self.log_abort()
      return res
  
  
    def __initPlugs__(self):
      if len( self.plugins ) > 0:
        l = []
        for p in self.plugins:
          mod = importlib.import_module( 'plugins.%s' % p )
          agent = mod.checker(self)
          l.append(agent)
        self.agents = tuple( l )
  
    def runChecks(self):
      """Run all the checks registered in this instance (in self.checks) and handle exceptions"""
  
      try:
        for c in self.checks:
          c()  # run check
        for agent in self.agents:
          agent.run() # run check
        self.completed = True
      except abortChecks:
        ## error logging done before raising this exception
        return
      except:
        self.log_exception( 'Exception caught by runChecks' )
        raise loggedException
      
  class checkFileName(checkBase):
    """Check basic syntax of file names (i.e. checks properties of the text string, it does not attempt to access the file).
  Inherits :class:`checkBase` class. Checks are run by the :meth:`check` method."""
  
    def init(self):
      self.id = 'C4.001'
      self.checkId = 'unset'
      self.isFixed = False
      self.step = 'Initialised'
      self.checks = (self.do_check_fn,self.do_check_fnextra)
      self.re_c1 = re.compile( '^[0-9]*$' )
      self.fnDict = {}
      self.fnTimeTuples = None
  ####
  
    def check(self,fn):
      """Initiate checks: manage arguments and then call *runChecks* (inherited from checkBase class).
    Arguments: fn: file name: the file name to be checked."""
      self.errorCount = 0
      assert type(fn) in [type('x'),type(u'x')], '1st argument to "check" method of checkGrids shound be a string variable name (not %s)' % type(fn)
      self.fn = fn
      self.fnsep = self.pcfg.fNameSep
  
      self.runChecks()
      self.parent.fnDict = self.fnDict
  ###
    def do_check_fn(self):
      """Basic file name checks:
         (1) Check suffix;
         (1b) [for ESA-CCI files] check presence of "ESACCI" and identify file naming convention;
         (2) Split file name into components and check number of such components;
         (3) Additional specialist checks for ESA-CCI, CORDEX, CMIP-type (for the time range).
      """
      fn = self.fn
      self.errorCount = 0
      self.completed = False
  
  ## check basic parsing of file name
      self.checkId = ('001','parse_filename')
      self.test( fn[-3:] == '.nc', 'File name ending ".nc" expected', abort=True, part=True )
      bits = string.split( fn[:-3], self.fnsep )
  
      self.fnParts = bits[:]
      if self.pcfg.domainIndex is not None:
        self.domain = self.fnParts[self.pcfg.domainIndex]
      else:
        self.domain = None
  
  
      if self.pcfg.projectV.id in ['ESA-CCI']:
        self.test( 'ESACCI' in bits[:2] or 'ESA' == bits[0], 'File name not a valid ESA-CCI file name: %s' % fn, abort=True )
        if bits[0] == 'ESA':
          self.esaFnId = 2
        elif bits[0] == 'ESACCI':
          self.esaFnId = 1
        else:
          self.esaFnId = 0
          bb = string.split( bits[2], '_' )
          self.test( bits[2][0] == 'L' and len(bb) == 2, 'Cannot parse ESA-CCI file name: %s' % fn, abort=True )
          bits = bits[:2] + bb + bits[3:]
          self.fnParts = bits[:]
          
        self.pcfg.setEsaCciFNType(self.esaFnId)
      self.test( len(bits) in self.pcfg.fnPartsOkLen, 'File name not parsed in %s elements [%s]' % (str(self.pcfg.fnPartsOkLen),str(bits)), abort=True )
  
      self.fnDict = {}
      if self.pcfg.projectV.id in ['ESA-CCI']:
        l0 = {0:6, 1:5, 2:5}[self.esaFnId]  
        for i in range(l0):
          x = self.pcfg.globalAttributesInFn[i]
          if x is not None and x[0] == '*':
            self.fnDict[x[1:]] = bits[i]
        self.fnDict['version'] = bits[-1]
        self.fnDict['gdsv'] = 'na'
        if self.esaFnId == 0:
          if len(bits) == 9:
            self.fnDict['additional'] = bits[-3]
            self.fnDict['gdsv'] = bits[-2]
          elif len(bits) == 8:
            if bits[-2][0] == 'v':
              self.fnDict['gdsv'] = bits[-2]
            else:
              self.fnDict['additional'] = bits[-2]
        elif self.esaFnId in [1,2]:
          if len(bits) == 8:
            self.fnDict['additional'] = bits[-3]
  
            
      
      if self.pcfg.groupIndex is not None:
        self.group = self.fnParts[self.pcfg.groupIndex]
      else:
        self.group = None
  
      if self.pcfg.freqIndex is not None:
        self.freq = self.fnParts[self.pcfg.freqIndex]
      elif self.group in ['fx','fixed']:
        self.freq = 'fx'
      else:
        self.freq = None
  
      ##if self.cls == 'CORDEX':
        ##self.freq = self.fnParts[7]
      ##elif self.cls == 'SPECS':
        ##self.freq = self.fnParts[1]
  
      self.var = self.fnParts[self.pcfg.varIndex]
  
      if self.pcfg.fnvdict is not None:
        if self.pcfg.fnvdict.has_key( self.var ):
          self.var = self.pcfg.fnvdict.get( self.var )['v']
        else:
          addi = self.fnDict.get('additional','xxxx')
          thiskey = '%s:%s' % (self.var,addi)
          if self.pcfg.fnvdict.has_key( thiskey ):
            self.var = self.pcfg.fnvdict.get( thiskey )['v']
  
      self.isFixed = self.freq in ['fx','fixed']
      self.parent.fileIsFixed = self.isFixed
      if self.isFixed:
        self.test( len(self.fnParts) in self.pcfg.fnPartsOkFixedLen, 'Number of file name elements not acceptable for fixed data' )
  
      self.checkId = ('002','parse_filename_timerange')
      if not self.isFixed:
  
  ## test time segment
        if self.pcfg.trangeType == 'CMIP':
          bits = string.split( self.fnParts[-1], '-' )
          self.test( len(bits) == 2, 'File time segment [%s] will not parse into 2 elements' % (self.fnParts[-1] ), abort=True, part=True )
  
          self.test(  len(bits[0]) == len(bits[1]), 'Start and end time specified in file name [%s] of unequal length' % (self.fnParts[-1] ), abort=True, part=True  )
          self.test(  int(bits[0]) <= int(bits[1]), 'Start and end time specified in file name [%s] in wrong order' % (self.fnParts[-1] ), abort=True, part=True  )
  
          for b in bits:
            self.test( self.isInt(b), 'Time segment in filename [%s] contains non integer characters' % (self.fnParts[-1] ),  abort=True, part=True  )
          self.log_pass()
          self.fnTimeParts = bits[:]
          l0 = []
          ## pick out month, day, hours etc, until end of string )
          for b in bits:
            l00 = [int( b[:4] ),]
            k0 = 4
            while k0 + 2 <= len(b):
              l00.append( int( b[k0:k0+2] ) )
              k0 += 2
            l0.append( tuple( l00 ) )
              
          self.fnTimeTuples = tuple( l0 )
  
        elif self.pcfg.trangeType == 'ESA-CCI':
          self.pcfg.checkTrangeLen = False
          tt = self.fnParts[self.pcfg.trangeIndex] 
          if self.test( len(tt) in [4,6,8,10,12,14] and self.re_c1.match(tt) is not None, 'Length of indicative date/time not consistent with YYYY[MM[DD[HH[MM[SS]]]]] specification: %s' % self.fnParts[-1], part=True  ):
            ll = [tt[:4],]
            tt = tt[4:]
            for j in range(5):
              if len(tt) > 0:
                ll.append( tt[:2] )
                tt = tt[2:]
              elif j in [1,2]:
                ll.append( '01' )
              else:
                ll.append( '00' )
            indDateTime = map( int, ll )
            self.test( indDateTime[1] in range(1,13), 'Invalid Month in indicative date time %s' % str(ll), part=True )
            self.test( indDateTime[2] in range(1,32), 'Invalid Day in indicative date time %s' % str(ll), part=True )
            self.test( indDateTime[3] in range(25), 'Invalid hour in indicative date time %s' % str(ll), part=True )
            self.test( indDateTime[4] in range(60), 'Invalid minute in indicative date time %s' % str(ll), part=True )
            self.test( indDateTime[5] in range(60), 'Invalid second in indicative date time %s' % str(ll), part=True )
  
      self.checkId = '003'
  
      self.checkId, ok = (('004','filename_timerange_length'),True)
      if (not self.isFixed) and self.pcfg.checkTrangeLen:
        ltr = { 'mon':6, 'sem':6, 'day':8, '3hr':[10,12], '6hr':10, '1hr':12 }
        ok &=self.test( self.freq in ltr.keys(), 'Frequency [%s] not recognised' % self.freq, part=True )
        if ok:
          if type( ltr[self.freq] ) == type(0):
            msg = 'Length of time range parts [%s,%s] not equal to required length [%s] for frequency %s' % (self.fnTimeParts[0],self.fnTimeParts[1],ltr[self.freq],self.freq)
            ok &= self.test( len(self.fnTimeParts[0]) == ltr[self.freq], msg, part=True )
          elif type( ltr[self.freq] ) in [type([]),type( () )]:
            msg = 'Length of time range parts [%s,%s] not in acceptable list [%s] for frequency %s' % (self.fnTimeParts[0],self.fnTimeParts[1],str(ltr[self.freq]),self.freq)
            ok &= self.test( len(self.fnTimeParts[0]) in ltr[self.freq], msg, part=True )
  
        if ok:
          self.log_pass()
  
    def do_check_fnextra(self):
      """Check whether file name components match constraints -- but only if those constraints are not implicitly verified through comparison with global attributes in later checks"""
      self.checkId = ('004','file_name_extra' )
      vocabs = self.pcfg.vocabs
      m = []
      for a in self.pcfg.controlledFnParts:
        if self.fnDict.has_key(a):
          try:
            if not vocabs[a].check( str(self.fnDict[a]) ):
              m.append( (a,self.fnDict[a],vocabs[a].note) )
          except:
            print 'failed trying to check file name component %s' % a
            raise baseException( 'failed trying to check file name component %s' % a )
  
      self.test( len(m)  == 0, 'File name components do not match constraints: %s' % str(m) )
  
  
  class checkGlobalAttributes(checkBase):
    """Check global and variable attributes, using tables of valid values"""
  
    def init(self):
      self.id = 'C4.002'
      self.checkId = 'unset'
      self.step = 'Initialised'
      self.checks = (self.do_check_ga,)
      self.fileId = None
  
    def check(self,globalAts, varAts,varName,varGroup, vocabs, fnParts):
      self.errorCount = 0
      assert type(varName) in [type('x'),type(u'x')], '1st argument to "check" method of checkGrids shound be a string variable name (not %s)' % type(varName)
      self.var = varName
      self.globalAts = globalAts
      self.varAts = varAts
      self.varGroup = varGroup
      self.vocabs = vocabs
      self.fnParts = fnParts
      self.runChecks()
  
    def getId(self):
      if self.fileId is None:
        id = self.globalAts['id']
  ##
  ## hack to allow for awkward usage in DLR ozone, where id is different for each file
  ## may need to improve on this if problem exists for other files in different form
  ##
        xx = re_dlr01.findall( id )
        if len(xx) != 0:
          id = id[:-19]
        if id != '':
          self.fileId = '%s.%s' % (self.globalAts['naming_authority'],id)
          if self.globalAts['naming_authority'] == 'uk.ac.pml':
            i0 = string.find(self.globalAts['id'],'OC4v6_QAA')
            if i0 != -1:
              self.fileId = '%s.%s' % (self.globalAts['naming_authority'],self.globalAts['id'][:i0+9])
        else:
          self.fileId = '%s.:%s:%s' % (self.globalAts['naming_authority'],self.globalAts['title'],self.globalAts['time_coverage_duration'])
  
    def getDrs( self ):
      assert self.completed, 'method getDrs should not be called if checks have not been completed successfully'
      ee = {}
      drsDefaults = { 'convention_version':'n/a'}
      if not self.globalAts.has_key('product'):
          self.globalAts['product'] = 'output'
      for k in self.drsMappings:
        if self.drsMappings[k] == '@var':
          ee[k] = self.var
        elif self.drsMappings[k][0] == '=':
          ee[k] = self.drsMappings[k][1:]
        elif self.drsMappings[k] == '@ensemble':
          ee[k] = "r%si%sp%s" % (self.globalAts["realization"],self.globalAts["initialization_method"],self.globalAts["physics_version"])
        elif self.drsMappings[k] == '@forecast_reference_time':
          x = self.globalAts.get("forecast_reference_time",'yyyy-mm-dd Thh:mm:ssZ' )
          ee[k] = "%s%s%s" % (x[:4],x[5:7],x[8:10])
        elif self.drsMappings[k] == '@mip_id':
          ee[k] = string.split( self.globalAts["table_id"] )[1]
        elif self.drsMappings[k] == '@ecv':
          ee[k] = self.pcfg.ecvMappings[ self.parent.fnDict['project'] ]
        elif self.drsMappings[k][0] == '$':
          self.pcfg.getExtraAtts()
          self.getId()
          if string.find(self.drsMappings[k],':') != -1:
            k2,dflt = string.split( self.drsMappings[k][1:],':')
            ee[k] = self.pcfg.extraAtts[self.fileId].get( k2, dflt )
          else:
            ee[k] = self.pcfg.extraAtts[self.fileId][self.drsMappings[k][1:]]
        elif self.drsMappings[k][0] == '*':
          thisk = self.drsMappings[k][1:]
          ee[k] = self.varAts[self.var].get(thisk,'__none__')
        elif self.drsMappings[k][0] == '#':
          thisk = self.drsMappings[k][1:]
          if drsDefaults.has_key( thisk ):
            ee[k] = self.parent.fnDict.get(thisk, drsDefaults[thisk] )
          else:
            ee[k] = self.parent.fnDict[thisk]
        else:
          ee[k] = self.globalAts[ self.drsMappings[k] ]
  
      for k in ['creation_date','tracking_id']:
        if k in self.globalAts.keys():
          ee[k] = self.globalAts[k]
  
      return ee
  
    def do_check_ga(self):
      varName = self.var
      globalAts = self.globalAts
      varAts = self.varAts
      varGroup = self.varGroup
      vocabs = self.vocabs
      fnParts = self.fnParts
  
      self.completed = False
      self.checkId = ('001','global_ncattribute_present')
      m = []
      for k in self.requiredGlobalAttributes:
        if not globalAts.has_key(k):
           m.append(k)
           self.globalAts[k] = '__errorReported__'
  
      if not self.test( len(m)  == 0, 'Required global attributes missing: %s' % str(m) ):
        gaerr = True
        for k in m:
          self.parent.amapListDraft.append( '#@;%s=%s|%s=%s' % (k,'__absent__',k,'<insert attribute value and uncomment>') )
  
      self.checkId = ('002','variable_in_group')
  
      
      self.test( varAts.has_key( varName ), 'Expected variable [%s] not present' % varName, abort=True, part=True )
      msg = 'Variable %s not in table %s' % (varName,varGroup)
      
      self.test( vocabs['variable'].isInTable( varName, varGroup ), msg, abort=True, part=True )
  
      if self.pcfg.checkVarType:
        self.checkId = ('003','variable_type')
  
        mipType = vocabs['variable'].getAttr( varName, varGroup, 'type' )
        thisType = {
          'real': 'float32',
          'integer': 'int32',
          'float': 'float32',
          'double': 'float64'
        }.get( mipType, mipType )
        variable_type = varAts[varName]['_type']
        if '>' in variable_type:
          # Big endian!
          variable_type = {'>f8': 'float64', '>f4': 'float32', '>i4': 'int32'}.get(variable_type, variable_type)
        self.test( mipType is None or variable_type == thisType, 'Variable [%s/%s] not of type %s [%s]' % (varName,varGroup,str(thisType),variable_type) )
      else:
        mipType = None
  
      self.checkId = ('004','variable_ncattribute_present')
      m = []
      reqAts = self.requiredVarAttributes[:]
      if (not self.parent.fileIsFixed) and self.pcfg.projectV.id in ['CORDEX']:
        reqAts.append( 'cell_methods' )
      for k in reqAts + vocabs['variable'].lists(varName, 'addRequiredAttributes'):
        if not varAts[varName].has_key(k):
           m.append(k)
      if not self.test( len(m)  == 0, 'Required variable attributes missing: %s' % str(m) ):
        vaerr = True
        for k in m:
          self.parent.amapListDraft.append( '#@var=%s;%s=%s|%s=%s' % (varName,k,'__absent__',k,'<insert attribute value and uncomment>') )
          ## print self.parent.amapListDraft[-1]
  
  ## need to insert a check that variable is present
      self.checkId = ('005','variable_ncattribute_mipvalues')
      ok = True
      hm = varAts[varName].get( 'missing_value', None ) is not None
      hf = varAts[varName].has_key( '_FillValue' )
      if hm or hf:
  ## both are not required for SPECS.
        if self.pcfg.varTables=='CMIP' and self.pcfg.projectV.id != 'SPECS':
          ok &= self.test( hm, 'missing_value must be present if _FillValue is [%s]' % varName )
          ok &= self.test( hf, '_FillValue must be present if missing_value is [%s]' % varName )
        else:
          ok = True
        if mipType == 'real':
          if varAts[varName].has_key( 'missing_value' ):
             msg = 'Variable [%s] has incorrect attribute missing_value=%s [correct: %s]' % (varName,varAts[varName]['missing_value'],self.missingValue)
  ### need to use ctypes here when using ncq3 to read files -- appears OK for other libraries.
             ok &= self.test( ctypes.c_float(varAts[varName]['missing_value']).value == ctypes.c_float(self.missingValue).value, msg, part=True )
          if varAts[varName].has_key( '_FillValue' ):
             msg = 'Variable [%s] has incorrect attribute _FillValue=%s [correct: %s]' % (varName,varAts[varName]['_FillValue'],self.missingValue)
             ok &= self.test( varAts[varName]['_FillValue'] == self.missingValue, msg, part=True )
  
      mm = []
      
      if self.pcfg.varTables=='CMIP':
        contAts = ['long_name', 'standard_name', 'units']
        if not self.parent.fileIsFixed:
        ##if varGroup not in ['fx','fixed']:
          contAts.append( 'cell_methods' )
      else:
        contAts = ['standard_name']
      hcm = varAts[varName].has_key( "cell_methods" )
      for k in contAts + vocabs['variable'].lists(varName,'addControlledAttributes'):
        targ = varAts[varName].get( k, 'Attribute not present' )
        val = vocabs['variable'].getAttr( varName, varGroup, k )
  
        if k == "standard_name":
          if val is not None:
            if val.find( ' ' ) != -1:
              val = string.join( string.split(val,maxsplit=1) )
            if targ.find( ' ' ) != -1:
              targ = string.join( string.split(targ,maxsplit=1) )
  
        if k == "cell_methods":
          if val is not None:
            parenthesies1 = []
            targ0 = targ[:]
            while string.find( targ, '(' ) != -1:
              i0 = targ.index( '(' )
              i1 = targ.index( ')' )
              parenthesies1.append( targ[i0:i1+1] )
              targ = targ[:i0-1] + targ[i1+1:]
            parenthesies2 = []
            val0 = val[:]
            while string.find( val, '(' ) != -1:
              i0 = val.index( '(' )
              i1 = val.index( ')' )
              parenthesies2.append( val[i0:i1+1] )
              val = val[:i0-1] + val[i1+1:]
            for p in parenthesies2:
              if p not in parenthesies1:
                mm.append( (k,parenthesies1,p) )
            if string.find( targ, val):
               mm.append( (k,targ,val) )
        elif targ != 'Attribute not present' and targ != val:
          mm.append( (k,targ,val) )
  
      ok &= self.test( len(mm)  == 0, 'Variable [%s] has incorrect attributes: %s' % (varName, strmm3(mm)), part=True )
      if len( mm  ) != 0:
        if self.parent.amapListDraft is None:
          self.parent.amapListDraft = []
        for m in mm:
            if m[2] is None:
              self.parent.amapListDraft.append( '## @var=%s;%s=%s|@delete=%s -- not supported yet' % (varName,m[0],m[1],m[0]) )
            else:
              self.parent.amapListDraft.append( '@var=%s;%s=%s|%s=%s' % (varName,m[0],m[1],m[0],m[2]) )
  
      if ok:
         self.log_pass()
  
      if (not self.parent.fileIsFixed) and hcm:
      ## if (varGroup not in ['fx','fixed']) and hcm:
        self.isInstantaneous = string.find( varAts[varName]['cell_methods'], 'time: point' ) != -1
      else:
        self.isInstantaneous = True
  
      self.checkId = ('006','global_ncattribute_cv' )
      m = []
      for a in self.controlledGlobalAttributes:
        if globalAts.has_key(a):
          try:
            if not vocabs[a].check( str(globalAts[a]) ):
              m.append( (a,globalAts[a],vocabs[a].note) )
          except:
            print 'failed trying to check global attribute %s' % a
            raise baseException( 'failed trying to check global attribute %s' % a )
  
      if not self.test( len(m)  == 0, 'Global attributes do not match constraints: %s' % str(m) ):
        for t in m:
          self.parent.amapListDraft.append( '#@;%s=%s|%s=%s' % (t[0],str(t[1]),t[0],'<insert attribute value and uncomment>' + str(t[2]) ) )
  
      self.checkId = ('007','filename_filemetadata_consistency')
      m = []
      for i in range(len(self.globalAttributesInFn)):
         gaif = self.globalAttributesInFn[i]
         if gaif is not None and gaif[0] != '*':
           if gaif[-1] == ':':
             bits = string.split(gaif,':')
             gaif0 = gaif
             gaif = bits[0]
             ix = int(bits[1])
           else:
             ix = i
  
           targVal = fnParts[ix]
           if gaif[0] == "@":
             if gaif[1:] == "mip_id":
                 bits = string.split( globalAts[ "table_id" ] ) 
                 if len( bits ) > 2 and bits[0] == "Table":
                   thisVal = bits[1]
                 else:
                   thisVal = globalAts[ "table_id" ]
                   self.test( False, 'Global attribute table_id does not conform to CMOR pattern ["Table ......"]: %s' % thisVal, part=True)
             elif gaif[1:] == "ensemble":
                 thisVal = "r%si%sp%s" % (globalAts["realization"],globalAts["initialization_method"],globalAts["physics_version"])
  ## following mappings are depricated -- introduced for SPECS and withdrawn ---
             elif gaif[1:] == "experiment_family":
                 thisVal = globalAts["experiment_id"][:-4]
             elif gaif[1:] == "forecast_reference_time":
                 x = self.globalAts.get("forecast_reference_time",'yyyy-mm-dd Thh:mm:ssZ' )
                 thisVal = "S%s%s%s" % (x[:4],x[5:7],x[8:10])
             elif gaif[1:] == "series":
                 thisVal = 'series%s' % globalAts["series"]
             else:
                 assert False, "Not coded to deal with this configuration: globalAttributesInFn[%s]=%s" % (i,self.globalAttributesInFn[i])
           
           else:
               thisVal = globalAts[gaif]
  
           if thisVal not in [targVal,'__errorReported__']:
               m.append( (i,self.globalAttributesInFn[i]) )
  
      self.test( len(m)  == 0,'File name segments do not match corresponding global attributes: %s' % str(m) )
  
      self.completed = True
         
  class checkStandardDims(checkBase):
    """Check the dimensions which are defined in the specifications"""
  
    def init(self):
      self.id = 'C4.003'
      self.checkId = 'unset'
      self.step = 'Initialised'
      self.checks = (self.check_timeVals, self.do_check,)
      ###self.plugins = ('timeAxisConsistency',)
      self.plevRequired = self.pcfg.plevRequired
      self.plevValues = self.pcfg.plevValues
      self.heightRequired = self.pcfg.heightRequired
      self.heightValues = self.pcfg.heightValues
      self.heightRange = self.pcfg.heightRange
  
    def check(self,varName,varGroup, da, va, isInsta,vocabs,fnParts):
      """API called from check manager *run_c4* for each file"""
  
      self.errorCount = 0
      assert type(varName) in [type('x'),type(u'x')], '1st argument to "check" method of checkGrids shound be a string variable name (not %s)' % type(varName)
      self.var = varName
      self.varGroup = varGroup
      self.da = da
      self.va = va
      self.fnParts = fnParts
      self.isInsta = isInsta
      self.vocabs = vocabs
      self.runChecks()
  
    def check_timeVals(self):
      da = self.da
      va = self.va
      self.errorCount = 0
      self.completed = False
      self.checkId = ('001','time_attributes')
      self.calendar = 'None'
      if not self.parent.fileIsFixed:
      ## if varGroup not in ['fx','fixed']:
        ok = True
        self.test( 'time' in da.keys(), 'Time dimension not found' , abort=True, part=True )
        if self.pcfg.varTables=='CMIP':
          if not self.isInsta:
            ok &= self.test(  da['time'].get( 'bounds', 'xxx' ) == 'time_bnds', 'Required bounds attribute not present or not correct value', part=True )
  
  ## is time zone designator needed?
          tunits = da['time'].get( 'units', 'xxx' )
          if self.project  == 'CORDEX':
            ok &= self.test( tunits in ["days since 1949-12-01 00:00:00Z", "days since 1949-12-01 00:00:00", "days since 1949-12-01"],
                 'Time units [%s] attribute not set correctly to "days since 1949-12-01 00:00:00Z"' % tunits, part=True )
          else:
            ok &= self.test( tunits[:10] == "days since", 'time units [%s] attribute not set correctly to "days since ....."' % tunits, part=True )
  
          ok &= self.test(  da['time'].has_key( 'calendar' ), 'Time: required attribute calendar missing', part=True )
  
          ok &= self.test( da['time']['_type'] in ["float64","double"], 'Time: data type not float64 [%s]' % da['time']['_type'], part=True )
         
          if ok:
            self.log_pass()
          self.calendar = da['time'].get( 'calendar', 'None' )
  
          if self.project  == 'CMIP5':
            pass
            ##print da['time']['_data'][0], da['time']['_data'][-1],self.parent.fnTimeTuples
      self.completed = True
  
    def do_check(self):
      varName = self.var
      varGroup = self.varGroup
      da = self.da
      va = self.va
      isInsta = self.isInsta
      self.errorCount = 0
      self.completed = False
  
  
      self.checkId = ('002','pressure_levels')
      if varName in self.plevRequired:
        ok = True
        self.test( 'plev' in va.keys(), 'plev coordinate not found %s' % str(va.keys()), abort=True, part=True )
  
        ok &= self.test( int( va['plev']['_data'][0] ) == self.plevValues[varName],  \
                    'plev value [%s] does not match required [%s]' % (va['plev']['_data'],self.plevValues[varName] ), part=True )
        
        plevAtDict = {'standard_name':"air_pressure", \
                      'long_name':"pressure", \
                      'units':"Pa", \
                      'positive':"down", \
                      'axis':"Z" }
  
        if varName in ['clh','clm','cll']:
          plevAtDict['bounds']= "plev_bnds"
  
        for k in plevAtDict.keys():
          ok &= self.test( va['plev'].get( k, None ) == plevAtDict[k], 
                       'plev attribute %s absent or wrong value (should be %s)' % (k,plevAtDict[k]), part=True )
  
        if varName in ['clh','clm','cll']:
           self.test( "plev_bnds" in va.keys(), 'plev_bnds variable not found %s' % str(va.keys()), abort=True, part=True )
           mm = []
           for k in plevAtDict.keys():
              if k != 'bounds' and k in va['plev_bnds'].keys():
                 if va['plev_bnds'][k] != va['plev'][k]:
                   mm.append(k)
           ok &= self.test( len(mm) == 0, 'Attributes of plev_bnds do not match those of plev: %s' % str(mm), part=True )
  
           bndsVals = {'clh':[44000, 0], 'clm':[68000, 44000], 'cll':[100000, 68000] }
           res = self.test( len( va['plev_bnds']['_data'] ) == 2, 'plev_bnds array is of wrong length', part=True )
           ok &= res
           if res:
              kk = 0
              for i in [0,1]:
                 if int(va['plev_bnds']['_data'][i]) != bndsVals[varName][i]:
                    kk+=1
              ok &= self.test( kk == 0, 'plev_bnds values not correct: should be %s' % str(bndsVals[varName]), part=True )
  
        if ok:
          self.log_pass()
  
      self.checkId = ('003','height_levels')
      hreq = varName in self.heightRequired
      if self.parent.experimental:
        print 'utils_c4: ', varName, self.vocabs['variable'].varcons[varGroup][varName].get( '_dimension',[])
        hreq = "height2m" in self.vocabs['variable'].varcons[varGroup][varName].get( '_dimension',[])
        if hreq:
          print 'testing height, var=%s' % varName
      if hreq:
        heightAtDict = {'long_name':"height", 'standard_name':"height", 'units':"m", 'positive':"up", 'axis':"Z" }
        ok = True
        ok &= self.test( 'height' in va.keys(), 'height coordinate not found %s' % str(va.keys()), abort=True, part=True )
        ##ok &= self.test( abs( va['height']['_data'] - self.heightValues[varName]) < 0.001, \
                  ##'height value [%s] does not match required [%s]' % (va['height']['_data'],self.heightValues[varName] ), part=True )
  
        ok1 = self.test( len( va['height']['_data'] ) == 1, 'More height values (%s) than expected (1)' % (len( va['height']['_data'])), part=True )
        if ok1:
          r = self.heightRange[varName]
          ok1 &= self.test( r[0] <= va['height']['_data'][0] <= r[1], \
                  'height value [%s] not in specified range [%s]' % (va['height']['_data'], (self.heightRange[varName] ) ), part=True )
  
        ok &= ok1
        
        for k in heightAtDict.keys():
          val =  va['height'].get( k, "none" )
          if not self.test( val == heightAtDict[k], \
                           'height attribute %s absent or wrong value (should be %s)' % (k,heightAtDict[k]), part=True ):
            self.parent.amapListDraft.append( '@var=%s;%s=%s|%s=%s' % ('height',k,val,k,heightAtDict[k]) )
            ok = False
  
        if ok:
          self.log_pass()
  
      self.completed = True
  
  class checkGrids(checkBase):
  
    def init(self):
      self.id = 'C4.004'
      self.checkId = 'unset'
      self.step = 'Initialised'
      self.checks = (self.do_check_rp,self.do_check_intd)
  
    def check(self,varName, domain, da, va):
      self.errorCount = 0
      assert type(varName) in [type('x'),type(u'x')], '1st argument to "check" method of checkGrids shound be a string variable name (not %s)' % type(varName)
      self.var = varName
      self.domain = domain
      self.da = da
      self.va = va
  
      self.runChecks()
      ##for c in self.checks:
        ##c()
      ##self.do_check_rp()
      ##self.do_check_intd()
  
    def do_check_rp(self):
      varName = self.var
      domain = self.domain
      da = self.da
      va = self.va
      if va[varName].get( 'grid_mapping', None ) == "rotated_pole":
        self.checkId = ('001','grid_mapping')
        atDict = { 'grid_mapping_name':'rotated_latitude_longitude' }
        atDict['grid_north_pole_latitude'] = self.pcfg.rotatedPoleGrids[domain]['grid_np_lat']
        if self.pcfg.rotatedPoleGrids[domain]['grid_np_lon'] != 'N/A':
          atDict['grid_north_pole_longitude'] = self.pcfg.rotatedPoleGrids[domain]['grid_np_lon']
  
        self.checkId = ('002','rotated_latlon_attributes')
        self.test( 'rlat' in da.keys() and 'rlon' in da.keys(), 'rlat and rlon not found (required for grid_mapping = rotated_pole )', abort=True, part=True )
  
        atDict = {'rlat':{'long_name':"rotated latitude", 'standard_name':"grid_latitude", 'units':"degrees", 'axis':"Y", '_type':'float64'},
                  'rlon':{'long_name':"rotated longitude", 'standard_name':"grid_longitude", 'units':"degrees", 'axis':"X", '_type':'float64'} }
        mm = []
        for k in ['rlat','rlon']:
          for k2 in atDict[k].keys():
            if atDict[k][k2] != da[k].get(k2, None ):
              mm.append( (k,k2) )
              record = '#@ax=%s;%s=%s|%s=%s <uncomment if correct>' % (k,k2,da[k].get(k2, '__missing__'),k2,atDict[k][k2]   )
              self.parent.amapListDraft.append( record )
        self.test( len(mm) == 0, 'Required attributes of grid coordinate arrays not correct: %s' % str(mm) )
  
        self.checkId = ('003','rotated_latlon_domain')
        ok = True
        for k in ['rlat','rlon']:
          res = len(da[k]['_data']) == self.pcfg.rotatedPoleGrids[domain][ {'rlat':'nlat','rlon':'nlon' }[k] ]
          if not res:
            self.test( res, 'Size of %s dimension does not match specification (%s,%s)' % (k,a,b), part=True  )
            ok = False
  
        a = ( da['rlat']['_data'][0], da['rlat']['_data'][-1], da['rlon']['_data'][0], da['rlon']['_data'][-1] )
        b = map( lambda x: self.pcfg.rotatedPoleGrids[domain][x], ['s','n','w','e'] )
        mm = []
        for i in range(4):
          if abs(a[i] - b[i]) > self.pcfg.gridSpecTol:
            mm.append( (a[i],b[i]) )
  
        ok &= self.test( len(mm) == 0, 'Domain boundaries for rotated pole grid do not match %s within tolerance (%s)' % (str(mm),self.pcfg.gridSpecTol), part=True )
  
        for k in ['rlat','rlon']:
          ok &= self.test( cs.check( da[k]['_data'] ), '%s values not evenly spaced -- min/max delta = %s, %s' % (k,cs.dmn,cs.dmx), part=True )
  
        if ok:
          self.log_pass()
  
    def do_check_intd(self):
      varName = self.var
      domain = self.domain
      da = self.da
      va = self.va
      if domain[-1] == 'i':
        self.checkId = ('004','regular_grid_attributes')
        self.test( 'lat' in da.keys() and 'lon' in da.keys(), 'lat and lon not found (required for interpolated data)', abort=True, part=True )
  
        atDict = {'lat':{'long_name':"latitude", 'standard_name':"latitude", 'units':"degrees_north", '_type':'float64'},
                  'lon':{'long_name':"longitude", 'standard_name':"longitude", 'units':"degrees_east", '_type':'float64'} }
        mm = []
        for k in ['lat','lon']:
          for k2 in atDict[k].keys():
            if atDict[k][k2] != da[k].get(k2, None ):
              mm.append( (k,k2) )
              record = '#@ax=%s;%s=%s|%s=%s <uncomment if correct>' % (k,k2,da[k].get(k2, '__missing__'),k2,atDict[k][k2]   )
              self.parent.amapListDraft.append( record )
  
        self.test( len(mm) == 0,  'Required attributes of grid coordinate arrays not correct: %s' % str(mm), part=True )
  
        ok = True
        self.checkId = ('005','regular_grid_domain')
        for k in ['lat','lon']:
          res = len(da[k]['_data']) >= self.pcfg.interpolatedGrids[domain][ {'lat':'nlat','lon':'nlon' }[k] ]
          if not res:
            a,b =  len(da[k]['_data']), self.pcfg.interpolatedGrids[domain][ {'lat':'nlat','lon':'nlon' }[k] ]
            self.test( res, 'Size of %s dimension does not match specification (%s,%s)' % (k,a,b), part=True )
            ok = False
  
        a = ( da['lat']['_data'][0], da['lat']['_data'][-1], da['lon']['_data'][0], da['lon']['_data'][-1] )
        b = map( lambda x: self.pcfg.interpolatedGrids[domain][x], ['s','n','w','e'] )
        rs = self.pcfg.interpolatedGrids[domain]['res']
        c = [-rs,rs,-rs,rs]
        mm = []
        for i in range(4):
          if a[i] != b[i]:
            x = (a[i]-b[i])/c[i]
            if x < 0 or abs( x - int(x) ) > 0.001:
               skipThis = False
               if self.project  == 'CORDEX':
                 if domain[:3] == 'ANT':
                   if i == 2 and abs( a[i] - 0.25 ) < 0.001:
                      skipThis = True
                   elif i == 3 and abs( a[i] - 359.75 ) < 0.001:
                      skipThis = True
               if not skipThis:
                 mm.append( (a[i],b[i]) )
  
        ok &= self.test( len(mm) == 0, 'Interpolated grid boundary error: File %s; Req. %s' % (str(a),str(b)), part=True )
  
        for k in ['lat','lon']:
          ok &= self.test( cs.check( da[k]['_data'] ), '%s values not evenly spaced -- min/max delta = %s, %s' % (k,cs.dmn,cs.dmx), part=True )
        if ok:
          self.log_pass()
  
  class checkByVar(checkBase):
    """Run some checks on groups of files with a common variable. Checks for continuity of time in group"""
  
    def init(self,fileNameSeparator='_'):
      self.id = 'C5.001'
      self.checkId = 'unset'
      self.step = 'Initialised'
      self.checks = (self.checkTrange,)
      self.fnsep = fileNameSeparator
  
    def setLogDict( self,fLogDict ):
      self.fLogDict = fLogDict
  
    def impt(self,flist):
      """Scan a list of files and identify groups which a common variable and extract time ranges into a dictionary of lists, keyed on group identifiers.
       :param flist: List of file names.
  
   This routine has rather obscure nested logical tests used to identify the group to which a file belongs. The complexity arises from the fact that the identification of the files that should form a continuous time series from the file names alone is not a standardised feature of the file names."""
      ee = {}
      elist = []
      for f in flist:
        fn = string.split(f, '/' )[-1]
        fnParts = string.split( fn[:-3], self.fnsep )
        
        try:
          if self.pcfg.freqIndex is not None:
            freq = fnParts[self.pcfg.freqIndex]
          else:
            freq = None
  
          group = fnParts[ self.pcfg.groupIndex ]
  
          if self.parent.fileIsFixed:
            trange = None
          else:
            trange = string.split( fnParts[-1], '-' )
          var = fnParts[self.pcfg.varIndex]
          thisKey = string.join( fnParts[:-1], '.' )
          if group not in ee.keys():
            ee[group] = {}
          if thisKey not in ee[group].keys():
            ee[group][thisKey] = []
          ee[group][thisKey].append( (f,fn,group,trange) )
        except:
          print 'Cannot parse file name: %s' % (f) 
          elist.append(f)
  ## this ee entry is not used, except in bookkeeping check below. 
  ## parsing of file name is repeated later, and a error log entry is created at that stage -- this could be improved.
  ## in order to improve, need to clarify flow of program: the list here is used to provide preliminary info before log files etc are set up.
          group = '__error__'
          thisKey = fn
          if group not in ee.keys():
            ee[group] = {}
          if thisKey not in ee[group].keys():
            ee[group][thisKey] = []
          ee[group][thisKey].append( (f,fn,group) )
  
      nn = len(flist)
      n2 = 0
      for k in ee.keys():
        for k2 in ee[k].keys():
          n2 += len( ee[k][k2] )
  
      assert nn==n2, 'some file lost!!!!!!'
      if len(elist) == 0:
        self.info =  '%s %s, %s' % (nn, maybe_plural("file", nn), str(ee.keys()))
      else:
        self.info =  '%s %s, %s frequencies, severe errors in file names: %s' % (nn, maybe_plural("file", nn), len(ee.keys()), len(elist))
        for e in elist:
          self.info += '\n%s' % e
      self.ee = ee
  
    def check(self, recorder=None,calendar='None',norun=False):
      self.errorCount = 0
      self.recorder=recorder
      self.calendar=calendar
  
      if calendar == '360_day':
        self.enddec = 30
      else:
        self.enddec = 31
      mm = { 'enddec':self.enddec }
      self.pats = {'mon':('(?P<d>[0-9]{3})101','(?P<e>[0-9]{3})012'), \
              'sem':('(?P<d>[0-9]{3})(012|101)','(?P<e>[0-9]{3})(011|010)'), \
              'day':('(?P<d>[0-9]{3}[16])0101','(?P<e>[0-9]{3}[50])12%(enddec)s' % mm), \
              'subd':('(?P<d>[0-9]{4})0101(?P<h1>[0-9]{2})(?P<mm>[30]0){0,1}$', '(?P<e>[0-9]{4})12%(enddec)s(?P<h2>[0-9]{2})([30]0){0,1}$' % mm ), \
              'subd2':('(?P<d>[0-9]{4})0101(?P<h1>[0-9]{2})', '(?P<e>[0-9]{4})010100' ) }
  
      if not norun:
        self.runChecks()
  
    def checkTrange(self):
      """Manage time range checks: loop over groups of files identified by :meth:`impt`"""
      keys = self.ee.keys()
      keys.sort()
      for k in keys:
        if k not in ['fx','fixed']:
          keys2 = self.ee[k].keys()
          keys2.sort()
          for k2 in keys2:
            self.checkThisTrange( self.ee[k][k2], k )
  
    def checkThisTrange( self, tt, group):
      """Check consistency across a list of time ranges"""
  
      if group in ['1hr','3hr','6hr']:
         kg = 'subd'
      else:
         kg = group
      ps = self.pats[kg]
      rere = (re.compile( ps[0] ), re.compile( ps[1] ) )
  
      n = len(tt)
      self.checkId = ('001','filename_timerange_value')
      for j in range(n):
        if self.monitor is not None:
           nofh0 = self.monitor.get_open_fds()
        t = tt[j]
        fn = t[1]
        isFirst = j == 0
        isLast = j == n-1
        lok = True
        for i in [0,1]:
          if not (i==0 and isFirst or i==1 and isLast):
            x = rere[i].match( t[3][i] )
            lok &= self.test( x is not None, 'Cannot match time range %s: %s [%s/%s]' % (i,fn,j,n), part=True, appendLogfile=(self.fLogDict.get(fn,None),fn) )
          if not lok:
            if self.recorder is not None:
              self.recorder.modify( t[1], 'ERROR: time range' )
        if self.monitor is not None:
           nofh9 = self.monitor.get_open_fds()
           if nofh9 > nofh0:
             print 'Open file handles: %s --- %s [%s]' % (nofh0, nofh9, j )
  
  
  ### http://stackoverflow.com/questions/2023608/check-what-files-are-open-in-python
  class sysMonitor(object):
  
    def __init__(self):
      self.fhCountMax = 0
  
    def get_open_fds(self):
      '''
      return the number of open file descriptors for current process
      .. warning: will only work on UNIX-like os-es.
      '''
      import subprocess
      import os
  
      pid = os.getpid()
      self.procs = subprocess.check_output( 
          [ "lsof", '-w', '-Ff', "-p", str( pid ) ] )
  
      self.ps = filter( 
              lambda s: s and s[ 0 ] == 'f' and s[1: ].isdigit(),
              self.procs.split( '\n' ) )
      self.fhCountMax = max( self.fhCountMax, len(self.ps) )
      return len( self.ps )
  
  def maybe_plural(word, n):
      "Return ``word`` if n is 1, and plural of ``word`` if n is not."
      if n == 1:
          return word
      return "%ss" % word
  