########################################################################
# File :    SiteDirector.py
# Author :  A.T., F.S.
########################################################################

"""  The Site Director is an agent performing pilot job submission to particular sites/Computing Elements.

.. literalinclude:: ../ConfigTemplate.cfg
  :start-after: ##BEGIN SiteDirector
  :end-before: ##END
  :dedent: 2
  :caption: SiteDirector options

"""

__RCSID__ = "$Id$"

import os
import sys
import random
import socket
import hashlib
from collections import defaultdict

import DIRAC
from DIRAC import S_OK, gConfig
from DIRAC.Core.Base.AgentModule import AgentModule
from DIRAC.Core.Utilities.SiteCEMapping import getSiteForCE
from DIRAC.Core.Utilities.Time import dateTime, second
from DIRAC.Core.Utilities.List import fromChar
from DIRAC.Core.Utilities.File import mkDir
from DIRAC.Core.Utilities.ObjectLoader import ObjectLoader
from DIRAC.ConfigurationSystem.Client.Helpers import CSGlobals, Registry
from DIRAC.ConfigurationSystem.Client.Helpers.Operations import Operations
from DIRAC.FrameworkSystem.Client.ProxyManagerClient import gProxyManager
from DIRAC.AccountingSystem.Client.Types.Pilot import Pilot as PilotAccounting
from DIRAC.AccountingSystem.Client.Types.PilotSubmission import PilotSubmission as PilotSubmissionAccounting
from DIRAC.AccountingSystem.Client.DataStoreClient import gDataStoreClient
from DIRAC.WorkloadManagementSystem.Client.MatcherClient import MatcherClient
from DIRAC.WorkloadManagementSystem.Client.ServerUtils import pilotAgentsDB
from DIRAC.WorkloadManagementSystem.Service.WMSUtilities import getGridEnv
from DIRAC.WorkloadManagementSystem.private.ConfigHelper import findGenericPilotCredentials
from DIRAC.WorkloadManagementSystem.Utilities.PilotWrapper import pilotWrapperScript, \
    _writePilotWrapperFile, getPilotFilesCompressedEncodedDict
from DIRAC.Resources.Computing.ComputingElementFactory import ComputingElementFactory
from DIRAC.ResourceStatusSystem.Client.ResourceStatus import ResourceStatus
from DIRAC.ResourceStatusSystem.Client.SiteStatus import SiteStatus

# dirac install file
DIRAC_INSTALL = os.path.join(DIRAC.rootPath, 'DIRAC', 'Core', 'scripts', 'dirac-install.py')

# pilot2 python files (NOT needed for pilot 3)
DIRAC_PILOT = os.path.join(DIRAC.rootPath, 'DIRAC', 'WorkloadManagementSystem', 'PilotAgent', 'dirac-pilot.py')
DIRAC_MODULES = [os.path.join(DIRAC.rootPath, 'DIRAC', 'WorkloadManagementSystem', 'PilotAgent', 'pilotCommands.py'),
                 os.path.join(DIRAC.rootPath, 'DIRAC', 'WorkloadManagementSystem', 'PilotAgent', 'pilotTools.py')]

# status
TRANSIENT_PILOT_STATUS = ['Submitted', 'Waiting', 'Running', 'Scheduled', 'Ready', 'Unknown']
WAITING_PILOT_STATUS = ['Submitted', 'Waiting', 'Scheduled', 'Ready']
FINAL_PILOT_STATUS = ['Aborted', 'Failed', 'Done']

MAX_PILOTS_TO_SUBMIT = 100
MAX_JOBS_IN_FILLMODE = 5


def getSubmitPools(group=None, vo=None):
  """ This method gets submit pools
  """
  if group:
    return Registry.getGroupOption(group, 'SubmitPools', '')
  if vo:
    return Registry.getVOOption(vo, 'SubmitPools', '')
  return ''


class SiteDirector(AgentModule):
  """ SiteDirector class provides an implementation of a DIRAC agent.

      Used for submitting pilots to Computing Elements.
  """

  def __init__(self, *args, **kwargs):
    """ c'tor
    """
    super(SiteDirector, self).__init__(*args, **kwargs)

    # on-the fly imports
    ol = ObjectLoader()
    res = ol.loadModule("ConfigurationSystem.Client.Helpers.Resources")
    if not res['OK']:
      sys.exit(res['Message'])
    self.resourcesModule = res['Value']

    self.queueDict = {}
    self.queueSlots = {}
    self.failedQueues = defaultdict(int)
    self.firstPass = True
    self.maxJobsInFillMode = MAX_JOBS_IN_FILLMODE
    self.maxPilotsToSubmit = MAX_PILOTS_TO_SUBMIT

    self.gridEnv = ''
    self.vo = ''
    self.group = ''
    # self.voGroups contain all the eligible user groups for pilots submitted by this SiteDirector
    self.voGroups = []
    self.pilotDN = ''
    self.pilotGroup = ''
    self.platforms = []
    self.sites = []
    self.defaultSubmitPools = ''
    self.totalSubmittedPilots = 0

    self.proxy = None

    self.addPilotsToEmptySites = False
    self.checkPlatform = False
    self.updateStatus = True
    self.getOutput = False
    self.sendAccounting = True
    self.sendSubmissionAccounting = True

    self.pilot3 = False
    self.pilotFiles = []  # files whose content will be compressed and sent together with the pilot wrapper

    self.siteClient = None
    self.rssClient = None
    self.rssFlag = None

    self.globalParameters = {"NumberOfProcessors": 1,
                             "MaxRAM": 2048}
    self.failedQueueCycleFactor = 10
    self.maxQueueLength = 86400 * 3

    self.pilotWaitingFlag = True
    self.pilotLogLevel = 'INFO'
    self.matcherClient = None
    self.siteMaskList = []
    self.ceMaskList = []

    self.localhost = socket.getfqdn()

  def initialize(self):
    """ Initial settings
    """

    self.gridEnv = self.am_getOption("GridEnv", '')
    if not self.gridEnv:
      self.gridEnv = getGridEnv()

    # The SiteDirector is for a particular user community
    self.vo = self.am_getOption("VO", '')
    if not self.vo:
      self.vo = self.am_getOption("Community", '')
    if not self.vo:
      self.vo = CSGlobals.getVO()
    # The SiteDirector is for a particular user group
    self.group = self.am_getOption("Group", '')

    # Choose the group for which pilots will be submitted. This is a hack until
    # we will be able to match pilots to VOs.
    if not self.group:
      if self.vo:
        result = Registry.getGroupsForVO(self.vo)
        if not result['OK']:
          return result
        self.voGroups = []
        for group in result['Value']:
          if 'NormalUser' in Registry.getPropertiesForGroup(group):
            self.voGroups.append(group)
    else:
      self.voGroups = [self.group]

    self.pilot3 = self.am_getOption('Pilot3', self.pilot3)

    # Get the clients
    self.siteClient = SiteStatus()
    self.rssClient = ResourceStatus()
    self.matcherClient = MatcherClient()

    return S_OK()

  def beginExecution(self):
    """ This is run at every cycle, as first thing.

        1. Check the pilots credentials.
        2. Get some flags and options used later
        3. Get the site description dictionary
        4. Get what to send in pilot wrapper
    """

    self.rssFlag = self.rssClient.rssFlag

    # Which credentials to use?
    # are they specific to the SD? (if not, get the generic ones)
    self.pilotDN = self.am_getOption("PilotDN", self.pilotDN)
    self.pilotGroup = self.am_getOption("PilotGroup", self.pilotGroup)
    result = findGenericPilotCredentials(vo=self.vo, pilotDN=self.pilotDN, pilotGroup=self.pilotGroup)
    if not result['OK']:
      return result
    self.pilotDN, self.pilotGroup = result['Value']

    # Parameters
    self.defaultSubmitPools = getSubmitPools(self.group, self.vo)
    self.workingDirectory = self.am_getOption('WorkDirectory')
    self.maxQueueLength = self.am_getOption('MaxQueueLength', self.maxQueueLength)
    self.pilotLogLevel = self.am_getOption('PilotLogLevel', self.pilotLogLevel)
    self.maxJobsInFillMode = self.am_getOption('MaxJobsInFillMode', self.maxJobsInFillMode)
    self.maxPilotsToSubmit = self.am_getOption('MaxPilotsToSubmit', self.maxPilotsToSubmit)
    self.pilotWaitingFlag = self.am_getOption('PilotWaitingFlag', self.pilotWaitingFlag)
    self.failedQueueCycleFactor = self.am_getOption('FailedQueueCycleFactor', self.failedQueueCycleFactor)
    self.pilotStatusUpdateCycleFactor = self.am_getOption('PilotStatusUpdateCycleFactor', 10)

    # Flags
    self.addPilotsToEmptySites = self.am_getOption('AddPilotsToEmptySites', self.addPilotsToEmptySites)
    self.checkPlatform = self.am_getOption('CheckPlatform', self.checkPlatform)
    self.updateStatus = self.am_getOption('UpdatePilotStatus', self.updateStatus)
    self.getOutput = self.am_getOption('GetPilotOutput', self.getOutput)
    self.sendAccounting = self.am_getOption('SendPilotAccounting', self.sendAccounting)
    self.sendSubmissionAccounting = self.am_getOption('SendPilotSubmissionAccounting', self.sendSubmissionAccounting)

    # Get the site description dictionary
    siteNames = None
    siteNamesOption = self.am_getOption('Site', ['any'])
    if siteNamesOption and 'any' not in [sn.lower() for sn in siteNamesOption]:
      siteNames = siteNamesOption

    ceTypes = None
    ceTypesOption = self.am_getOption('CETypes', ['any'])
    if ceTypesOption and 'any' not in [ct.lower() for ct in ceTypesOption]:
      ceTypes = ceTypesOption

    ces = None
    cesOption = self.am_getOption('CEs', ['any'])
    if cesOption and 'any' not in [ce.lower() for ce in cesOption]:
      ces = cesOption

    self.log.always('VO:', self.vo)
    if self.voGroups:
      self.log.always('Group(s):', self.voGroups)
    self.log.always('Sites:', siteNames)
    self.log.always('CETypes:', ceTypes)
    self.log.always('CEs:', ces)
    self.log.always('PilotDN:', self.pilotDN)
    self.log.always('PilotGroup:', self.pilotGroup)

    result = self.resourcesModule.getQueues(community=self.vo,
                                            siteList=siteNames,
                                            ceList=ces,
                                            ceTypeList=ceTypes,
                                            mode='Direct')
    if not result['OK']:
      return result
    result = self.getQueues(result['Value'])
    if not result['OK']:
      return result

    if self.updateStatus:
      self.log.always('Pilot status update requested')
    if self.getOutput:
      self.log.always('Pilot output retrieval requested')
    if self.sendAccounting:
      self.log.always('Pilot accounting sending requested')
    if self.sendSubmissionAccounting:
      self.log.always('Pilot submission accounting sending requested')

    self.log.always('MaxPilotsToSubmit:', self.maxPilotsToSubmit)
    self.log.always('MaxJobsInFillMode:', self.maxJobsInFillMode)

    if self.firstPass:
      if self.queueDict:
        self.log.always("Agent will serve queues:")
        for queue in self.queueDict:
          self.log.always("Site: %s, CE: %s, Queue: %s" % (self.queueDict[queue]['Site'],
                                                           self.queueDict[queue]['CEName'],
                                                           queue))
    self.firstPass = False

    # which files to send in the pilotWrapper?
    if not self.pilot3:
      # If it's pilot 3, nothing will be sent: the wrapper will get them all
      self.pilotFiles = DIRAC_MODULES + [DIRAC_PILOT]

    return S_OK()

  def __generateQueueHash(self, queueDict):
    """ Generate a hash of the queue description
    """
    myMD5 = hashlib.md5()
    myMD5.update(str(queueDict))
    hexstring = myMD5.hexdigest()
    return hexstring

  def getQueues(self, resourceDict):
    """ Get the list of relevant CEs (what is in resourceDict) and their descriptions.

        The main goal of this method is to create self.queueDict
    """

    self.log.debug("Getting queues this SD will submit to")
    self.log.debug("The resources dictionary contains %d entries" % len(resourceDict))

    self.queueDict = {}
    queueCECache = {}
    ceFactory = ComputingElementFactory()

    for site in resourceDict:
      for ce in resourceDict[site]:
        ceDict = resourceDict[site][ce]
        pilotRunDirectory = ceDict.get('PilotRunDirectory', '')
        # ceMaxRAM = ceDict.get('MaxRAM', None)
        qDict = ceDict.pop('Queues')
        for queue in qDict:
          queueName = '%s_%s' % (ce, queue)
          self.queueDict[queueName] = {}
          self.queueDict[queueName]['ParametersDict'] = qDict[queue]
          self.queueDict[queueName]['ParametersDict']['Queue'] = queue
          self.queueDict[queueName]['ParametersDict']['Site'] = site
          self.queueDict[queueName]['ParametersDict']['GridEnv'] = self.gridEnv
          self.queueDict[queueName]['ParametersDict']['Setup'] = gConfig.getValue('/DIRAC/Setup', 'unknown')
          # Evaluate the CPU limit of the queue according to the Glue convention
          # To Do: should be a utility
          if "maxCPUTime" in self.queueDict[queueName]['ParametersDict'] and \
             "SI00" in self.queueDict[queueName]['ParametersDict']:
            maxCPUTime = float(self.queueDict[queueName]['ParametersDict']['maxCPUTime'])
            # For some sites there are crazy values in the CS
            maxCPUTime = max(maxCPUTime, 0)
            maxCPUTime = min(maxCPUTime, 86400 * 12.5)
            si00 = float(self.queueDict[queueName]['ParametersDict']['SI00'])
            queueCPUTime = 60. / 250. * maxCPUTime * si00
            self.queueDict[queueName]['ParametersDict']['CPUTime'] = int(queueCPUTime)

          # Tags & RequiredTags defined on the Queue level and on the CE level are concatenated
          # This also converts them from a string to a list if required.
          for tagFieldName in ('Tag', 'RequiredTag'):
            ceTags = ceDict.get(tagFieldName, [])
            if isinstance(ceTags, basestring):
              ceTags = fromChar(ceTags)
            queueTags = self.queueDict[queueName]['ParametersDict'].get(tagFieldName)
            if queueTags and isinstance(queueTags, basestring):
              queueTags = fromChar(queueTags)
              self.queueDict[queueName]['ParametersDict'][tagFieldName] = queueTags
            if ceTags:
              if queueTags:
                allTags = list(set(ceTags + queueTags))
                self.queueDict[queueName]['ParametersDict'][tagFieldName] = allTags
              else:
                self.queueDict[queueName]['ParametersDict'][tagFieldName] = ceTags

          # Some parameters can be defined on the CE level and are inherited by all Queues
          for parameter in ['MaxRAM', 'NumberOfProcessors', 'WholeNode']:
            queueParameter = self.queueDict[queueName]['ParametersDict'].get(parameter)
            ceParameter = ceDict.get(parameter)
            if ceParameter or queueParameter:
              self.queueDict[queueName]['ParametersDict'][parameter] = ceParameter if not queueParameter \
                  else queueParameter

          if pilotRunDirectory:
            self.queueDict[queueName]['ParametersDict']['JobExecDir'] = pilotRunDirectory
          qwDir = os.path.join(self.workingDirectory, queue)
          mkDir(qwDir)
          self.queueDict[queueName]['ParametersDict']['WorkingDirectory'] = qwDir
          ceQueueDict = dict(ceDict)
          ceQueueDict.update(self.queueDict[queueName]['ParametersDict'])

          # Generate the CE object for the queue or pick the already existing one
          # if the queue definition did not change
          queueHash = self.__generateQueueHash(ceQueueDict)
          if queueName in queueCECache and queueCECache[queueName]['Hash'] == queueHash:
            queueCE = queueCECache[queueName]['CE']
          else:
            result = ceFactory.getCE(ceName=ce,
                                     ceType=ceDict['CEType'],
                                     ceParametersDict=ceQueueDict)
            if not result['OK']:
              return result
            queueCECache.setdefault(queueName, {})
            queueCECache[queueName]['Hash'] = queueHash
            queueCECache[queueName]['CE'] = result['Value']
            queueCE = queueCECache[queueName]['CE']

          self.queueDict[queueName]['ParametersDict'].update(queueCE.ceParameters)
          self.queueDict[queueName]['CE'] = queueCE
          self.queueDict[queueName]['CEName'] = ce
          self.queueDict[queueName]['CEType'] = ceDict['CEType']
          self.queueDict[queueName]['Site'] = site
          self.queueDict[queueName]['QueueName'] = queue
          self.queueDict[queueName]['QueryCEFlag'] = ceDict.get('QueryCEFlag', "false")

          if self.checkPlatform:
            platform = ''
            if "Platform" in self.queueDict[queueName]['ParametersDict']:
              platform = self.queueDict[queueName]['ParametersDict']['Platform']
            elif "Platform" in ceDict:
              platform = ceDict['Platform']
            elif "OS" in ceDict:
              architecture = ceDict.get('architecture', 'x86_64')
              platform = '_'.join([architecture, ceDict['OS']])
            if platform and platform not in self.platforms:
              self.platforms.append(platform)

            if "Platform" not in self.queueDict[queueName]['ParametersDict'] and platform:
              result = self.resourcesModule.getDIRACPlatform(platform)
              if result['OK']:
                self.queueDict[queueName]['ParametersDict']['Platform'] = result['Value'][0]
            self.queueDict[queueName]['Platform'] = platform

          result = self.queueDict[queueName]['CE'].isValid()
          if not result['OK']:
            self.log.fatal(result['Message'])
            return result
          if 'BundleProxy' in self.queueDict[queueName]['ParametersDict']:
            if self.queueDict[queueName]['ParametersDict']['BundleProxy'].lower() in ['true', 'yes', '1']:
              self.queueDict[queueName]['BundleProxy'] = True
          elif 'BundleProxy' in ceDict:
            if ceDict['BundleProxy'].lower() in ['true', 'yes', '1']:
              self.queueDict[queueName]['BundleProxy'] = True

          if site not in self.sites:
            self.sites.append(site)

          if "WholeNode" in self.queueDict[queueName]['ParametersDict']:
            self.globalParameters['WholeNode'] = 'True'
          for parameter in ['MaxRAM', 'NumberOfProcessors']:
            if parameter in self.queueDict[queueName]['ParametersDict']:
              self.globalParameters[parameter] = max(self.globalParameters[parameter],
                                                     int(self.queueDict[queueName]['ParametersDict'][parameter]))

    return S_OK()

  def execute(self):
    """ Main execution method (what is called at each agent cycle).

        It basically just calls self.submitPilots() method
    """

    if not self.queueDict:
      self.log.warn('No site defined, exiting the cycle')
      return S_OK()

    # get list of usable sites within this cycle
    result = self.siteClient.getUsableSites()
    if not result['OK']:
      return result
    self.siteMaskList = result.get('Value', [])

    if self.rssFlag:
      ceNamesList = [queue['CEName'] for queue in self.queueDict.itervalues()]
      result = self.rssClient.getElementStatus(ceNamesList, "ComputingElement")
      if not result['OK']:
        self.log.error("Can not get the status of computing elements",
                       " %s" % result['Message'])
        return result
      self.ceMaskList = [ceName for ceName in result['Value'].iterkeys() if result['Value'][ceName]
                         ['all'] in ('Active', 'Degraded')]

    result = self.submitPilots()
    if not result['OK']:
      self.log.error('Errors in the job submission: ', result['Message'])
      return result

    # Every N cycles we update the pilots status
    cyclesDone = self.am_getModuleParam('cyclesDone')
    if self.updateStatus and cyclesDone % self.pilotStatusUpdateCycleFactor == 0:
      result = self.updatePilotStatus()
      if not result['OK']:
        self.log.error('Errors in updating pilot status: ', result['Message'])
        return result

    return S_OK()

  def submitPilots(self):
    """ Go through defined computing elements and submit pilots if necessary and possible

        :return: S_OK/S_ERROR
    """

    # First, we check if we want to submit pilots at all, and also where
    submit, anySite, jobSites, testSites = self._ifAndWhereToSubmit()
    if not submit:
      self.log.notice("Not submitting any pilots at this cycle")
      return S_OK()

    # From here on we assume we are going to (try to) submit some pilots
    self.log.debug("Going to try to submit some pilots")

    self.log.verbose("Queues treated", ','.join(self.queueDict))

    self.totalSubmittedPilots = 0

    queueDictItems = list(self.queueDict.items())
    random.shuffle(queueDictItems)

    for queueName, queueDictionary in queueDictItems:
      # now submitting to the single queues
      self.log.verbose("Evaluating queue", queueName)

      # are we going to submit pilots to this specific queue?
      if not self._allowedToSubmit(queueName, anySite, jobSites, testSites):
        continue

      if 'CPUTime' in queueDictionary['ParametersDict']:
        queueCPUTime = int(queueDictionary['ParametersDict']['CPUTime'])
      else:
        self.log.warn('CPU time limit is not specified, skipping',
                      'queue %s' % queueName)
        continue
      if queueCPUTime > self.maxQueueLength:
        queueCPUTime = self.maxQueueLength

      ce, ceDict = self._getCE(queueName)

      # additionalInfo is normally taskQueueDict
      pilotsWeMayWantToSubmit, additionalInfo = self._getPilotsWeMayWantToSubmit(ceDict)
      self.log.debug('%d pilotsWeMayWantToSubmit are eligible for %s queue' % (pilotsWeMayWantToSubmit, queueName))
      if not pilotsWeMayWantToSubmit:
        self.log.debug('...so skipping %s' % queueName)
        continue

      # Get the number of already waiting pilots for the queue
      totalWaitingPilots = 0
      manyWaitingPilotsFlag = False
      if self.pilotWaitingFlag:
        tqIDList = additionalInfo.keys()
        result = pilotAgentsDB.countPilots({'TaskQueueID': tqIDList,
                                            'Status': WAITING_PILOT_STATUS},
                                           None)
        if not result['OK']:
          self.log.error('Failed to get Number of Waiting pilots', result['Message'])
          totalWaitingPilots = 0
        else:
          totalWaitingPilots = result['Value']
          self.log.debug('Waiting Pilots: %s' % totalWaitingPilots)
      if totalWaitingPilots >= pilotsWeMayWantToSubmit:
        self.log.verbose("Possibly enough pilots already waiting", '(%d)' % totalWaitingPilots)
        manyWaitingPilotsFlag = True
        if not self.addPilotsToEmptySites:
          continue

      self.log.debug("%d waiting pilots for the total of %d eligible pilots for %s" %
                     (totalWaitingPilots, pilotsWeMayWantToSubmit, queueName))

      # Get the number of available slots on the target site/queue
      totalSlots = self.getQueueSlots(queueName, manyWaitingPilotsFlag)
      if totalSlots == 0:
        self.log.debug('%s: No slots available' % queueName)
        continue

      if manyWaitingPilotsFlag:
        # Throttle submission of extra pilots to empty sites
        pilotsToSubmit = self.maxPilotsToSubmit / 10 + 1
      else:
        pilotsToSubmit = max(0, min(totalSlots, pilotsWeMayWantToSubmit - totalWaitingPilots))
        self.log.info('%s: Slots=%d, TQ jobs(pilotsWeMayWantToSubmit)=%d, Pilots: waiting %d, to submit=%d' %
                      (queueName, totalSlots, pilotsWeMayWantToSubmit, totalWaitingPilots, pilotsToSubmit))

      # Limit the number of pilots to submit to MAX_PILOTS_TO_SUBMIT
      pilotsToSubmit = min(self.maxPilotsToSubmit, pilotsToSubmit)

      # Get the working proxy
      cpuTime = queueCPUTime + 86400
      self.log.verbose("Getting pilot proxy",
                       "for %s/%s %d long" % (self.pilotDN, self.pilotGroup, cpuTime))
      result = gProxyManager.getPilotProxyFromDIRACGroup(self.pilotDN, self.pilotGroup, cpuTime)
      if not result['OK']:
        return result
      self.proxy = result['Value']
      # Check returned proxy lifetime
      result = self.proxy.getRemainingSecs()  # pylint: disable=no-member
      if not result['OK']:
        return result
      lifetime_secs = result['Value']
      ce.setProxy(self.proxy, lifetime_secs)

      # now really submitting
      while pilotsToSubmit:  # a cycle because pilots are submitted in chunks
        res = self._submitPilotsToQueue(pilotsToSubmit, ce, queueName)
        if not res['OK']:
          self.log.info("Won't try further because of failures", "Queue: %s" % queueName)
          pilotsToSubmit = 0
          pilotList = []
          stampDict = {}
        else:
          pilotsToSubmit, pilotList, stampDict = res['Value']

        # updating the pilotAgentsDB... done by default but maybe not strictly necessary
        res = self._addPilotTQReference(queueName, additionalInfo, pilotList, stampDict)

    self.log.info("Total number of pilots submitted in this cycle", '%d' % self.totalSubmittedPilots)

    return S_OK()

  def _ifAndWhereToSubmit(self):
    """ Return a tuple that says if and where to submit pilots:

        (submit, anySite, jobSites, testSites)
        e.g.
        (True, False, {'Site1', 'Site2'}, {'Test1', 'Test2'})

        VOs may want to replace this method with different strategies
    """

    tqDict = self._getTQDictForMatching()
    if not tqDict:
      return True, True, set(), set()

    # the tqDict used here is a very generic one, not specific to one CE/queue only
    self.log.verbose('Checking overall TQ availability with requirements')
    self.log.verbose(tqDict)

    # Check that there is some work at all
    result = self.matcherClient.getMatchingTaskQueues(tqDict)
    if not result['OK']:
      self.log.error("Matcher error:", result['Message'])
      return False, True, set(), set()
    matchingTQs = result['Value']
    if not matchingTQs:
      self.log.notice(
          'No Waiting jobs suitable for the director, so nothing to submit')
      return False, True, set(), set()

    # If we are here there's some work to do, now let's see for where
    jobSites = set()
    testSites = set()
    anySite = False

    for tqDescription in matchingTQs.itervalues():
      siteList = tqDescription.get('Sites', [])
      if siteList:
        jobSites |= set(siteList)
      else:
        anySite = True

      if "JobTypes" in tqDescription:
        if "Sites" in tqDescription:
          for site in tqDescription['Sites']:
            if site.lower() != 'any':
              testSites.add(site)

    self.monitorJobsQueuesPilots(matchingTQs)

    return True, anySite, jobSites, testSites

  def monitorJobsQueuesPilots(self, matchingTQs):
    """ Just printout of jobs queues and pilots status in TQ
    """
    tqIDList = matchingTQs.keys()
    result = pilotAgentsDB.countPilots({'TaskQueueID': tqIDList,
                                        'Status': WAITING_PILOT_STATUS},
                                       None)

    totalWaitingJobs = 0
    for tqDescription in matchingTQs.itervalues():
      totalWaitingJobs += tqDescription['Jobs']

    if not result['OK']:
      self.log.error("Can't count pilots", result['Message'])
    else:
      self.log.info("Total jobs : number of task queues : number of waiting pilots",
                    "%d : %d : %d" % (totalWaitingJobs, len(tqIDList), result['Value']))

  def _getTQDictForMatching(self):
    """ Just construct a dictionary (tqDict)
        that will be used to check with Matcher if there's anything to submit.

        If extensions want, they can replace partly or fully this method.
        If it returns just an empty dict, the assuption is that we'll submit pilots no matters what.

        :returns dict: tqDict of task queue descriptions
    """
    tqDict = {'Setup': CSGlobals.getSetup(),
              'CPUTime': 9999999,
              'SubmitPool': self.defaultSubmitPools}
    if self.vo:
      tqDict['Community'] = self.vo
    if self.voGroups:
      tqDict['OwnerGroup'] = self.voGroups

    if self.checkPlatform:
      platforms = self._getPlatforms()
      if platforms:
        tqDict['Platform'] = platforms

    tqDict['Site'] = self.sites

    # Get a union of all tags
    tags = []
    for queue in self.queueDict:
      tags += self.queueDict[queue]['ParametersDict'].get('Tag', [])
    tqDict['Tag'] = list(set(tags))

    # Add overall max values for all queues
    tqDict.update(self.globalParameters)

    return tqDict

  def _getPlatforms(self):
    """ Get the platforms used for TQ match
        Here for extension purpose.

        :return: list of platforms
    """
    result = self.resourcesModule.getCompatiblePlatforms(self.platforms)
    if not result['OK']:
      self.log.error("Issue getting compatible platforms, will skip check of platforms",
                     self.platforms + " : " + result['Message'])
    return result['Value']

  def _allowedToSubmit(self, queue, anySite, jobSites, testSites):
    """ Check if we are allowed to submit to a certain queue

        :param queue: the queue name
        :type queue: basestring
        :param anySite: submitting anywhere?
        :type anySite: bool
        :param jobSites: set of job site names (only considered if anySite is False)
        :type jobSites: set
        :param testSites: set of test site names
        :type testSites: set

        :return: True/False
    """

    # Check if the queue failed previously
    failedCount = self.failedQueues[queue] % self.failedQueueCycleFactor
    if failedCount != 0:
      self.log.warn("queue failed recently ==> number of cycles skipped",
                    "%s ==> %d" % (queue, 10 - failedCount))
      self.failedQueues[queue] += 1
      return False

    # Check the status of the site
    if self.queueDict[queue]['Site'] not in self.siteMaskList\
            and self.queueDict[queue]['Site'] not in testSites:
      self.log.verbose("Queue skipped (site not in mask)",
                       "%s (%s)" % (self.queueDict[queue]['QueueName'],
                                    self.queueDict[queue]['Site']))
      return False

    # Check that there are task queues waiting for this site
    if not anySite and self.queueDict[queue]['Site'] not in jobSites:
      self.log.verbose("Queue skipped: no workload expected",
                       "%s at %s" % (self.queueDict[queue]['QueueName'],
                                     self.queueDict[queue]['Site']))
      return False

    # Check the status of the CE (only for RSS=Active)
    if self.rssFlag:
      if self.queueDict[queue]['CEName'] not in self.ceMaskList:
        self.log.verbose("Skipping computing element: resource not usable",
                         "%s at %s" % (self.queueDict[queue]['CEName'],
                                       self.queueDict[queue]['Site']))
        return False

    # if we are here, it means that we are allowed to submit to the queue
    return True

  def _getCE(self, queue):
    """ Prepare the queue description to look for eligible jobs

        :param queue: queue name
        :type queue: basestring

        :return: ce (ComputingElement object), ceDict (dict)
    """

    ce = self.queueDict[queue]['CE']
    ceDict = ce.getParameterDict()
    ceDict['GridCE'] = self.queueDict[queue]['CEName']

    if self.queueDict[queue]['Site'] not in self.siteMaskList:
      ceDict['JobType'] = "Test"
    if self.vo:
      ceDict['Community'] = self.vo
    if self.voGroups:
      ceDict['OwnerGroup'] = self.voGroups

    # This is a hack to get rid of !
    ceDict['SubmitPool'] = self.defaultSubmitPools

    if self.checkPlatform:
      result = self.resourcesModule.getCompatiblePlatforms(self.queueDict[queue]['Platform'])
      if not result['OK']:
        self.log.error("Issue getting compatible platforms, returning 'ANY'",
                       "%s: %s" % (self.platforms, result['Message']))
        ceDict['Platform'] = 'ANY'
      ceDict['Platform'] = result['Value']

    return ce, ceDict

  def _getPilotsWeMayWantToSubmit(self, ceDict):
    """ Returns the number of pilots that we may want to submit to the ce described in ceDict

        This implementation is based on the number of eligible WMS taskQueues for the target site/queue.
        VOs are free to override this method and to provide a different implementation.

        :param ceDict: dictionary describing CE
        :type ceDict: dict

        :return: pilotsWeMayWantToSubmit (int), taskQueueDict (dict)
        :rType: tuple
    """

    pilotsWeMayWantToSubmit = 0

    result = self.matcherClient.getMatchingTaskQueues(ceDict)
    if not result['OK']:
      self.log.error('Could not retrieve TaskQueues from TaskQueueDB', result['Message'])
      return result
    taskQueueDict = result['Value']
    if not taskQueueDict:
      self.log.verbose('No matching TQs found',
                       'for %s' % ceDict)

    for tq in taskQueueDict.itervalues():
      pilotsWeMayWantToSubmit += tq['Jobs']

    return pilotsWeMayWantToSubmit, taskQueueDict

  def _submitPilotsToQueue(self, pilotsToSubmit, ce, queue):
    """ Method that really submits the pilots to the ComputingElements' queue

       :param pilotsToSubmit: number of pilots to submit. Maybe only part of this amount will be submitted here.
       :type pilotsToSubmit: int
       :param ce: computing element object to where we submit
       :type ce: ComputingElement
       :param queue: queue where to submit
       :type queue: basestring

       :return: S_OK/S_ERROR.
                If S_OK, returns tuple with (pilotsToSubmit, pilotList, stampDict)
                where
                  pilotsToSubmit is the pilots still to submit (maybe 0)
                  pilotsList is the list of pilots submitted
                  stampDict is a dict of timestamps of pilots submission
       :rtype: dict
    """
    self.log.info('Going to submit pilots',
                  "(a maximum of %d pilots to %s queue)" % (pilotsToSubmit, queue))

    bundleProxy = self.queueDict[queue].get('BundleProxy', False)
    jobExecDir = self.queueDict[queue]['ParametersDict'].get('JobExecDir', '')
    envVariables = self.queueDict[queue]['ParametersDict'].get('EnvironmentVariables', None)

    executable, pilotSubmissionChunk = self.getExecutable(queue, pilotsToSubmit,
                                                          bundleProxy=bundleProxy,
                                                          jobExecDir=jobExecDir,
                                                          envVariables=envVariables)

    submitResult = ce.submitJob(executable, '', pilotSubmissionChunk)
    # FIXME: The condor thing only transfers the file with some
    # delay, so when we unlink here the script is gone
    # FIXME 2: but at some time we need to clean up the pilot wrapper scripts...
    if not (self.queueDict[queue]['CEType'] == 'HTCondorCE' or
            (self.queueDict[queue]['CEType'] == 'Local' and ce.batchSystem == 'Condor')):
      os.unlink(executable)
    if not submitResult['OK']:
      self.log.error("Failed submission to queue",
                     "Queue %s:\n, %s" % (queue, submitResult['Message']))

      if self.sendSubmissionAccounting:
        self.sendPilotSubmissionAccounting(self.queueDict[queue]['Site'],
                                           self.queueDict[queue]['CEName'],
                                           self.queueDict[queue]['QueueName'],
                                           pilotsToSubmit,
                                           0,
                                           'Failed')

      pilotsToSubmit = 0
      self.failedQueues[queue] += 1
      return submitResult

    pilotsToSubmit = pilotsToSubmit - pilotSubmissionChunk
    # Add pilots to the PilotAgentsDB: assign pilots to TaskQueue proportionally to the task queue priorities
    pilotList = submitResult['Value']
    self.queueSlots[queue]['AvailableSlots'] -= len(pilotList)
    self.totalSubmittedPilots += len(pilotList)
    self.log.info('Submitted %d pilots to %s@%s' % (len(pilotList),
                                                    self.queueDict[queue]['QueueName'],
                                                    self.queueDict[queue]['CEName']))
    stampDict = submitResult.get('PilotStampDict', {})
    if self.sendSubmissionAccounting:
      self.sendPilotSubmissionAccounting(self.queueDict[queue]['Site'],
                                         self.queueDict[queue]['CEName'],
                                         self.queueDict[queue]['QueueName'],
                                         len(pilotList),
                                         len(pilotList),
                                         'Succeeded')

    return S_OK((pilotsToSubmit, pilotList, stampDict))

  def _addPilotTQReference(self, queue, taskQueueDict, pilotList, stampDict):
    """ Add to pilotAgentsDB the reference of for which TqID the pilots have been sent

        :param queue: the queue name
        :type queue: basestring
        :param taskQueueDict: dict of task queues
        :type taskQueueDict: dict
        :param pilotList: list of pilots
        :type pilotList: list
        :param stampDict: dictionary of pilots timestamps
        :type stampDict: dict

        :return: None
    """

    tqPriorityList = []
    sumPriority = 0.
    for tq in taskQueueDict:
      sumPriority += taskQueueDict[tq]['Priority']
      tqPriorityList.append((tq, sumPriority))
    tqDict = {}
    for pilotID in pilotList:
      rndm = random.random() * sumPriority
      for tq, prio in tqPriorityList:
        if rndm < prio:
          tqID = tq
          break
      if tqID not in tqDict:
        tqDict[tqID] = []
      tqDict[tqID].append(pilotID)

    for tqID, pilotsList in tqDict.iteritems():
      result = pilotAgentsDB.addPilotTQReference(pilotsList,
                                                 tqID,
                                                 self.pilotDN,
                                                 self.pilotGroup,
                                                 self.localhost,
                                                 self.queueDict[queue]['CEType'],
                                                 stampDict)
      if not result['OK']:
        self.log.error('Failed add pilots to the PilotAgentsDB', result['Message'])
        continue
      for pilot in pilotsList:
        result = pilotAgentsDB.setPilotStatus(pilot,
                                              'Submitted',
                                              self.queueDict[queue]['CEName'],
                                              'Successfully submitted by the SiteDirector',
                                              self.queueDict[queue]['Site'],
                                              self.queueDict[queue]['QueueName'])
        if not result['OK']:
          self.log.error('Failed to set pilot status', result['Message'])
          continue

  def getQueueSlots(self, queue, manyWaitingPilotsFlag):
    """ Get the number of available slots in the queue
    """
    ce = self.queueDict[queue]['CE']
    ceName = self.queueDict[queue]['CEName']
    queueName = self.queueDict[queue]['QueueName']
    queryCEFlag = self.queueDict[queue]["QueryCEFlag"].lower() in ["1", "yes", "true"]

    self.queueSlots.setdefault(queue, {})
    totalSlots = self.queueSlots[queue].get('AvailableSlots', 0)

    # See if there are waiting pilots for this queue. If not, allow submission
    if totalSlots and manyWaitingPilotsFlag:
      result = pilotAgentsDB.selectPilots({'DestinationSite': ceName,
                                           'Queue': queueName,
                                           'Status': WAITING_PILOT_STATUS})
      if result['OK']:
        jobIDList = result['Value']
        if not jobIDList:
          return totalSlots
      return 0

    availableSlotsCount = self.queueSlots[queue].setdefault('AvailableSlotsCount', 0)
    waitingJobs = 1
    if totalSlots == 0:
      if availableSlotsCount % 10 == 0:

        # Get the list of already existing pilots for this queue
        jobIDList = None
        result = pilotAgentsDB.selectPilots({'DestinationSite': ceName,
                                             'Queue': queueName,
                                             'Status': TRANSIENT_PILOT_STATUS})

        if result['OK']:
          jobIDList = result['Value']

        if queryCEFlag:
          result = ce.available(jobIDList)
          if not result['OK']:
            self.log.warn('Failed to check the availability of queue',
                          '%s: \n%s' % (queue, result['Message']))
            self.failedQueues[queue] += 1
          else:
            ceInfoDict = result['CEInfoDict']
            self.log.info("CE queue report",
                          "(%s_%s): Wait=%d, Run=%d, Submitted=%d, Max=%d" %
                          (ceName, queueName, ceInfoDict['WaitingJobs'], ceInfoDict['RunningJobs'],
                           ceInfoDict['SubmittedJobs'], ceInfoDict['MaxTotalJobs']))
            totalSlots = result['Value']
            self.queueSlots[queue]['AvailableSlots'] = totalSlots
            waitingJobs = ceInfoDict['WaitingJobs']
        else:
          maxWaitingJobs = int(self.queueDict[queue]['ParametersDict'].get('MaxWaitingJobs', 10))
          maxTotalJobs = int(self.queueDict[queue]['ParametersDict'].get('MaxTotalJobs', 10))
          waitingToRunningRatio = float(self.queueDict[queue]['ParametersDict'].get('WaitingToRunningRatio', 0.0))
          waitingJobs = 0
          totalJobs = 0
          if jobIDList:
            result = pilotAgentsDB.getPilotInfo(jobIDList)
            if not result['OK']:
              self.log.warn("Failed to check PilotAgentsDB",
                            "for queue %s: \n%s" % (queue, result['Message']))
              self.failedQueues[queue] += 1
            else:
              for _pilotRef, pilotDict in result['Value'].iteritems():
                if pilotDict["Status"] in TRANSIENT_PILOT_STATUS:
                  totalJobs += 1
                  if pilotDict["Status"] in WAITING_PILOT_STATUS:
                    waitingJobs += 1
              runningJobs = totalJobs - waitingJobs
              self.log.info("PilotAgentsDB report",
                            "(%s_%s): Wait=%d, Run=%d, Max=%d" %
                            (ceName, queueName, waitingJobs, runningJobs, maxTotalJobs))
              maxWaitingJobs = int(max(maxWaitingJobs, runningJobs * waitingToRunningRatio))

          totalSlots = min((maxTotalJobs - totalJobs), (maxWaitingJobs - waitingJobs))
          self.queueSlots[queue]['AvailableSlots'] = max(totalSlots, 0)

    self.queueSlots[queue]['AvailableSlotsCount'] += 1

    if manyWaitingPilotsFlag and waitingJobs:
      return 0
    return totalSlots

#####################################################################################
  def getExecutable(self, queue, pilotsToSubmit,
                    bundleProxy=True, jobExecDir='', envVariables=None,
                    **kwargs):
    """ Prepare the full executable for queue

    :param queue: queue name
    :type queue: basestring
    :param pilotsToSubmit: number of pilots to submit
    :type pilotsToSubmit: int
    :param bundleProxy: flag that say if to bundle or not the proxy
    :type bundleProxy: bool
    :param queue: pilot execution dir (normally an empty string)
    :type queue: basestring

    :returns: a string the options for the pilot
    :rtype: basestring
    """

    proxy = None
    if bundleProxy:
      proxy = self.proxy
    pilotOptions, pilotsSubmitted = self._getPilotOptions(queue, pilotsToSubmit, **kwargs)
    if not pilotOptions:
      self.log.warn("Pilots will be submitted without additional options")
      pilotOptions = []
    if not pilotsSubmitted:
      pilotsSubmitted = pilotsToSubmit
    pilotOptions = ' '.join(pilotOptions)
    self.log.verbose('pilotOptions: %s' % pilotOptions)
    executable = self._writePilotScript(workingDirectory=self.workingDirectory,
                                        pilotOptions=pilotOptions,
                                        proxy=proxy,
                                        pilotExecDir=jobExecDir,
                                        envVariables=envVariables)
    return executable, pilotsSubmitted

#####################################################################################

  def _getPilotOptions(self, queue, pilotsToSubmit, **kwargs):
    """ Prepare pilot options

    :param queue: queue name
    :type queue: basestring
    :param pilotsToSubmit: number of pilots to submit
    :type pilotsToSubmit: int

    :returns: pilotOptions, pilotsToSubmit tuple where
              pilotOptions is a list of strings, each one is an option to the dirac-pilot script invocation
              pilotsToSubmit is the number of pilots to submit
    :rtype: tuple
    """
    queueDict = self.queueDict[queue]['ParametersDict']
    pilotOptions = []

    setup = gConfig.getValue("/DIRAC/Setup", "unknown")
    if setup == 'unknown':
      self.log.error('Setup is not defined in the configuration')
      return [None, None]
    pilotOptions.append('-S %s' % setup)
    opsHelper = Operations(group=self.pilotGroup, setup=setup)

    # Installation defined?
    installationName = opsHelper.getValue("Pilot/Installation", "")
    if installationName:
      pilotOptions.append('-V %s' % installationName)

    # Project defined?
    projectName = opsHelper.getValue("Pilot/Project", "")
    if projectName:
      pilotOptions.append('-l %s' % projectName)
    else:
      self.log.info('DIRAC project will be installed by pilots')

    # Request a release
    # FIXME: this can disapper at some point (when there will only be pilot 3)
    if not self.pilot3:  # in pilot 3 the version is taken from the JSON file exported from the CS
      diracVersion = opsHelper.getValue("Pilot/Version", [])
      if not diracVersion:
        self.log.error('Pilot/Version is not defined in the configuration')
        return [None, None]
      # diracVersion is a list of accepted releases
      pilotOptions.append('-r %s' % ','.join(str(it) for it in diracVersion))

    # lcgBundle defined?
    lcgBundleVersion = opsHelper.getValue("Pilot/LCGBundleVersion", "")
    if lcgBundleVersion:
      self.log.warn(
          "lcgBundle defined in CS: will overwrite possible per-release lcg bundle versions",
          "(version in CS: %s)" % lcgBundleVersion)
      pilotOptions.append('-g %s' % lcgBundleVersion)

    # DIRACOS defined?
    # FIXME: this can disapper at some point
    diracOS = opsHelper.getValue("Pilot/DIRACOS", False)
    if diracOS:
      self.log.warn("DIRACOS forced with CS option: will overwrite possible per-release lcg bundle versions")
      pilotOptions.append('--dirac-os')

    ownerDN = self.pilotDN
    ownerGroup = self.pilotGroup
    # Request token for maximum pilot efficiency
    result = gProxyManager.requestToken(ownerDN, ownerGroup, pilotsToSubmit * self.maxJobsInFillMode)
    if not result['OK']:
      self.log.error('Invalid proxy token request', result['Message'])
      return [None, None]
    (token, numberOfUses) = result['Value']
    pilotOptions.append('-o /Security/ProxyToken=%s' % token)
    # Use Filling mode
    pilotOptions.append('-M %s' % min(numberOfUses, self.maxJobsInFillMode))

    # Since each pilot will execute min( numberOfUses, self.maxJobsInFillMode )
    # with numberOfUses tokens we can submit at most:
    #    numberOfUses / min( numberOfUses, self.maxJobsInFillMode )
    # pilots
    newPilotsToSubmit = numberOfUses / min(numberOfUses, self.maxJobsInFillMode)
    if newPilotsToSubmit != pilotsToSubmit:
      self.log.info("Number of pilots to submit is changed",
                    "to %d after getting the proxy token" % newPilotsToSubmit)
      pilotsToSubmit = newPilotsToSubmit
    # Debug
    if self.pilotLogLevel.lower() == 'debug':
      pilotOptions.append('-ddd')
    # CS Servers
    csServers = gConfig.getValue("/DIRAC/Configuration/Servers", [])
    pilotOptions.append('-C %s' % ",".join(csServers))

    # DIRAC Extensions to be used in pilots
    pilotExtensionsList = opsHelper.getValue("Pilot/Extensions", [])
    extensionsList = []
    if pilotExtensionsList:
      if pilotExtensionsList[0] != 'None':
        extensionsList = pilotExtensionsList
    else:
      extensionsList = [ext for ext in CSGlobals.getCSExtensions() if 'Web' not in ext]
    if extensionsList:
      pilotOptions.append('-e %s' % ",".join(extensionsList))

    # CEName
    pilotOptions.append('-N %s' % self.queueDict[queue]['CEName'])
    # Queue
    pilotOptions.append('-Q %s' % self.queueDict[queue]['QueueName'])
    # SiteName
    pilotOptions.append('-n %s' % queueDict['Site'])
    if 'ClientPlatform' in queueDict:
      pilotOptions.append("-p '%s'" % queueDict['ClientPlatform'])

    if 'SharedArea' in queueDict:
      pilotOptions.append("-o '/LocalSite/SharedArea=%s'" % queueDict['SharedArea'])

    if "ExtraPilotOptions" in queueDict:
      pilotOptions.append(queueDict['ExtraPilotOptions'])

    # Hack
    if self.defaultSubmitPools:
      pilotOptions.append('-o /Resources/Computing/CEDefaults/SubmitPool=%s' % self.defaultSubmitPools)

    if self.group:
      pilotOptions.append('-G %s' % self.group)

    return pilotOptions, pilotsToSubmit

####################################################################################

  def _writePilotScript(self, workingDirectory, pilotOptions,
                        proxy=None,
                        pilotExecDir='',
                        envVariables=None):
    """ Bundle together and write out the pilot executable script, admix the proxy if given

     :param workingDirectory: pilot wrapper working directory
     :type workingDirectory: basestring
     :param pilotOptions: options with which to start the pilot
     :type pilotOptions: basestring
     :param proxy: proxy file we are going to bundle
     :type proxy: basestring
     :param pilotExecDir: pilot executing directory
     :type pilotExecDir: basestring

     :returns: file name of the pilot wrapper created
     :rtype: basestring
    """

    try:
      pilotFilesCompressedEncodedDict = getPilotFilesCompressedEncodedDict(self.pilotFiles + [DIRAC_INSTALL],
                                                                           proxy)
    except BaseException as be:
      self.log.exception("Exception during pilot modules files compression", lException=be)

    location = ''
    if self.pilot3:
      location = Operations().getValue("Pilot/pilotFileServer")
    localPilot = pilotWrapperScript(pilotFilesCompressedEncodedDict=pilotFilesCompressedEncodedDict,
                                    pilotOptions=pilotOptions,
                                    pilotExecDir=pilotExecDir,
                                    envVariables=envVariables,
                                    location=location)

    return _writePilotWrapperFile(workingDirectory=workingDirectory, localPilot=localPilot)

  def updatePilotStatus(self):
    """ Update status of pilots in transient states
    """
    for queue in self.queueDict:
      ce = self.queueDict[queue]['CE']
      ceName = self.queueDict[queue]['CEName']
      queueName = self.queueDict[queue]['QueueName']
      ceType = self.queueDict[queue]['CEType']
      siteName = self.queueDict[queue]['Site']

      result = pilotAgentsDB.selectPilots({'DestinationSite': ceName,
                                           'Queue': queueName,
                                           'GridType': ceType,
                                           'GridSite': siteName,
                                           'Status': TRANSIENT_PILOT_STATUS,
                                           'OwnerDN': self.pilotDN,
                                           'OwnerGroup': self.pilotGroup})
      if not result['OK']:
        self.log.error('Failed to select pilots", ": %s' % result['Message'])
        continue
      pilotRefs = result['Value']
      if not pilotRefs:
        continue

      result = pilotAgentsDB.getPilotInfo(pilotRefs)
      if not result['OK']:
        self.log.error('Failed to get pilots info from DB', result['Message'])
        continue
      pilotDict = result['Value']

      stampedPilotRefs = []
      for pRef in pilotDict:
        if pilotDict[pRef]['PilotStamp']:
          stampedPilotRefs.append(pRef + ":::" + pilotDict[pRef]['PilotStamp'])
        else:
          stampedPilotRefs = list(pilotRefs)
          break

      # This proxy is used for checking the pilot status and renewals
      # We really need at least a few hours otherwise the renewed
      # proxy may expire before we check again...
      result = ce.isProxyValid(3 * 3600)
      if not result['OK']:
        result = gProxyManager.getPilotProxyFromDIRACGroup(self.pilotDN, self.pilotGroup, 23400)
        if not result['OK']:
          return result
        self.proxy = result['Value']
        ce.setProxy(self.proxy, 23300)

      result = ce.getJobStatus(stampedPilotRefs)
      if not result['OK']:
        self.log.error('Failed to get pilots status from CE', '%s: %s' % (ceName, result['Message']))
        continue
      pilotCEDict = result['Value']

      abortedPilots, getPilotOutput = self._updatePilotStatus(pilotRefs, pilotDict, pilotCEDict)
      for pRef in getPilotOutput:
        self._getPilotOutput(pRef, pilotDict, ce, ceName)

      # If something wrong in the queue, make a pause for the job submission
      if abortedPilots:
        self.failedQueues[queue] += 1

    # The pilot can be in Done state set by the job agent check if the output is retrieved
    for queue in self.queueDict:
      ce = self.queueDict[queue]['CE']

      if not ce.isProxyValid(120)['OK']:
        result = gProxyManager.getPilotProxyFromDIRACGroup(self.pilotDN, self.pilotGroup, 1000)
        if not result['OK']:
          return result
        self.proxy = result['Value']
        ce.setProxy(self.proxy, 940)

      ceName = self.queueDict[queue]['CEName']
      queueName = self.queueDict[queue]['QueueName']
      ceType = self.queueDict[queue]['CEType']
      siteName = self.queueDict[queue]['Site']
      result = pilotAgentsDB.selectPilots({'DestinationSite': ceName,
                                           'Queue': queueName,
                                           'GridType': ceType,
                                           'GridSite': siteName,
                                           'OutputReady': 'False',
                                           'Status': FINAL_PILOT_STATUS})

      if not result['OK']:
        self.log.error('Failed to select pilots', result['Message'])
        continue
      pilotRefs = result['Value']
      if not pilotRefs:
        continue
      result = pilotAgentsDB.getPilotInfo(pilotRefs)
      if not result['OK']:
        self.log.error('Failed to get pilots info from DB', result['Message'])
        continue
      pilotDict = result['Value']
      if self.getOutput:
        for pRef in pilotRefs:
          self._getPilotOutput(pRef, pilotDict, ce, ceName)

      # Check if the accounting is to be sent
      if self.sendAccounting:
        result = pilotAgentsDB.selectPilots({'DestinationSite': ceName,
                                             'Queue': queueName,
                                             'GridType': ceType,
                                             'GridSite': siteName,
                                             'AccountingSent': 'False',
                                             'Status': FINAL_PILOT_STATUS})

        if not result['OK']:
          self.log.error('Failed to select pilots', result['Message'])
          continue
        pilotRefs = result['Value']
        if not pilotRefs:
          continue
        result = pilotAgentsDB.getPilotInfo(pilotRefs)
        if not result['OK']:
          self.log.error('Failed to get pilots info from DB', result['Message'])
          continue
        pilotDict = result['Value']
        result = self.sendPilotAccounting(pilotDict)
        if not result['OK']:
          self.log.error('Failed to send pilot agent accounting')

    return S_OK()

  def _updatePilotStatus(self, pilotRefs, pilotDict, pilotCEDict):
    """ Really updates the pilots status

        :return: number of aborted pilots, flag for getting the pilot output
    """

    abortedPilots = 0
    getPilotOutput = []

    for pRef in pilotRefs:
      newStatus = ''
      oldStatus = pilotDict[pRef]['Status']
      lastUpdateTime = pilotDict[pRef]['LastUpdateTime']
      sinceLastUpdate = dateTime() - lastUpdateTime

      ceStatus = pilotCEDict.get(pRef, oldStatus)

      if oldStatus == ceStatus and ceStatus != "Unknown":
        # Normal status did not change, continue
        continue
      elif ceStatus == oldStatus == "Unknown":
        if sinceLastUpdate < 3600 * second:
          # Allow 1 hour of Unknown status assuming temporary problems on the CE
          continue
        else:
          newStatus = 'Aborted'
      elif ceStatus == "Unknown" and oldStatus not in FINAL_PILOT_STATUS:
        # Possible problems on the CE, let's keep the Unknown status for a while
        newStatus = 'Unknown'
      elif ceStatus != 'Unknown':
        # Update the pilot status to the new value
        newStatus = ceStatus

      if newStatus:
        self.log.info('Updating status',
                      'to %s for pilot %s' % (newStatus, pRef))
        result = pilotAgentsDB.setPilotStatus(pRef, newStatus, '', 'Updated by SiteDirector')
        if not result['OK']:
          self.log.error(result['Message'])
        if newStatus == "Aborted":
          abortedPilots += 1
      # Set the flag to retrieve the pilot output now or not
      if newStatus in FINAL_PILOT_STATUS:
        if pilotDict[pRef]['OutputReady'].lower() == 'false' and self.getOutput:
          getPilotOutput.append(pRef)

    return abortedPilots, getPilotOutput

  def _getPilotOutput(self, pRef, pilotDict, ce, ceName):
    """ Retrieves the pilot output for a pilot and stores it in the pilotAgentsDB
    """

    self.log.info('Retrieving output for pilot %s' % pRef)
    pilotStamp = pilotDict[pRef]['PilotStamp']
    pRefStamp = pRef
    if pilotStamp:
      pRefStamp = pRef + ':::' + pilotStamp
    result = ce.getJobOutput(pRefStamp)
    if not result['OK']:
      self.log.error('Failed to get pilot output',
                     '%s: %s' % (ceName, result['Message']))
    else:
      output, error = result['Value']
      if output:
        result = pilotAgentsDB.storePilotOutput(pRef, output, error)
        if not result['OK']:
          self.log.error('Failed to store pilot output', result['Message'])
      else:
        self.log.warn('Empty pilot output not stored to PilotDB')

  def sendPilotAccounting(self, pilotDict):
    """ Send pilot accounting record
    """
    for pRef in pilotDict:
      self.log.verbose('Preparing accounting record',
                       'for pilot %s' % pRef)
      pA = PilotAccounting()
      pA.setEndTime(pilotDict[pRef]['LastUpdateTime'])
      pA.setStartTime(pilotDict[pRef]['SubmissionTime'])
      retVal = Registry.getUsernameForDN(pilotDict[pRef]['OwnerDN'])
      if not retVal['OK']:
        userName = 'unknown'
        self.log.error("Can't determine username for dn", pilotDict[pRef]['OwnerDN'])
      else:
        userName = retVal['Value']
      pA.setValueByKey('User', userName)
      pA.setValueByKey('UserGroup', pilotDict[pRef]['OwnerGroup'])
      result = getSiteForCE(pilotDict[pRef]['DestinationSite'])
      if result['OK'] and result['Value'].strip():
        pA.setValueByKey('Site', result['Value'].strip())
      else:
        pA.setValueByKey('Site', 'Unknown')
      pA.setValueByKey('GridCE', pilotDict[pRef]['DestinationSite'])
      pA.setValueByKey('GridMiddleware', pilotDict[pRef]['GridType'])
      pA.setValueByKey('GridResourceBroker', pilotDict[pRef]['Broker'])
      pA.setValueByKey('GridStatus', pilotDict[pRef]['Status'])
      if 'Jobs' not in pilotDict[pRef]:
        pA.setValueByKey('Jobs', 0)
      else:
        pA.setValueByKey('Jobs', len(pilotDict[pRef]['Jobs']))
      self.log.verbose("Adding accounting record",
                       "for pilot %s" % pilotDict[pRef]['PilotID'])
      retVal = gDataStoreClient.addRegister(pA)
      if not retVal['OK']:
        self.log.error('Failed to send accounting info for pilot ', pRef)
      else:
        # Set up AccountingSent flag
        result = pilotAgentsDB.setAccountingFlag(pRef)
        if not result['OK']:
          self.log.error('Failed to set accounting flag for pilot ', pRef)

    self.log.info('Committing accounting records',
                  'for %d pilots' % len(pilotDict))
    result = gDataStoreClient.commit()
    if result['OK']:
      for pRef in pilotDict:
        self.log.verbose('Setting AccountingSent flag',
                         'for pilot %s' % pRef)
        result = pilotAgentsDB.setAccountingFlag(pRef)
        if not result['OK']:
          self.log.error('Failed to set accounting flag for pilot ', pRef)
    else:
      return result

    return S_OK()

  def sendPilotSubmissionAccounting(self,
                                    siteName,
                                    ceName,
                                    queueName,
                                    numTotal,
                                    numSucceeded,
                                    status):

    """ Send pilot submission accounting record
        :param str siteName:     Site name
        :param str ceName:       CE name
        :param str queueName:    queue Name
        :param int numTotal:     Total number of submission
        :param int numSucceeded: Total number of submission succeeded
        :param str status:       'Succeeded' or 'Failed'
    """

    pA = PilotSubmissionAccounting()
    pA.setStartTime(dateTime())
    pA.setEndTime(dateTime())
    pA.setValueByKey('HostName', DIRAC.siteName())
    if hasattr(self, "_AgentModule__moduleProperties"):
      pA.setValueByKey('SiteDirector', self.am_getModuleParam('agentName'))
    else:  # In case it is not executed as agent
      pA.setValueByKey('SiteDirector', 'Client')

    pA.setValueByKey('Site', siteName)
    pA.setValueByKey('CE', ceName)
    pA.setValueByKey('Queue', queueName)
    pA.setValueByKey('Status', status)
    pA.setValueByKey('NumTotal', numTotal)
    pA.setValueByKey('NumSucceeded', numSucceeded)
    result = gDataStoreClient.addRegister(pA)

    if not result['OK']:
      self.log.warn('Error in add Register:' + result['Message'])
      return result

    self.log.verbose("Begin commit")
    result = gDataStoreClient.delayedCommit()
    if not result['OK']:
      self.log.error('Error in Commit:' + result['Message'])
      return result
