text: # (C) Copyright 1996-2018 ECMWF.
  # 
  # This software is licensed under the terms of the Apache Licence Version 2.0
  # which can be obtained at http://www.apache.org/licenses/LICENSE-2.0. 
  # In applying this licence, ECMWF does not waive the privileges and immunities 
  # granted to it by virtue of its status as an intergovernmental organisation nor
  # does it submit to any jurisdiction.
  
  import csv
  import os
  from datetime import datetime
  
  from kronos_modeller.kronos_exceptions import ConfigurationError
  from kronos_modeller.jobs import IngestedJob, ModelJob
  
  from kronos_modeller.logreader.dataset import IngestedDataSet
  
  
  def read_pbs_log(filename_in):
  
      pbs_jobs = []
  
      log_list_raw = ['ctime',
                      'qtime',
                      'etime',
                      'end',
                      'start',
                      'resources_used.ncpus',
                      'Resource_List.ncpus',
                      'resources_used.mem',
                      'resources_used.cpupercent',
                      'resources_used.cput',
                      'group',
                      'jobname',
                      'Resource_List.EC_nodes',
                      'queue'
                      ]
  
      # - standardize the key nemes for different PBS log labels..
      log_list = ['time_created',
                  'time_queued',
                  'time_eligible',
                  'time_end',
                  'time_start',
                  'ncpus',
                  'ncpus',
                  'memory_kb',
                  'cpu_percent',
                  'cpu_percent',
                  'group',
                  'jobname',
                  'nnodes',
                  'queue'
                  ]
  
      # Read file
      f_in = open(filename_in, "r")
      cc = 1
      cce = 0
      for line in f_in:
  
          # array got by splitting the line
          larray = line.split(" ")
          b1_array = larray[1].split(";")
  
          # block E
          if (b1_array[1] == "E"):
  
              # init dictionary
              line_dict = {}
  
              # user name
              user_name = b1_array[3].split("=")[1]
              line_dict["user"] = str(user_name)
  
              for jobL in range(0, len(larray)):
  
                  yval_val = larray[jobL].split("=")
                  # print yval_val
  
                  if (len(yval_val) == 2):
                      if yval_val[0] in log_list_raw:
                          # find index
                          idx = log_list_raw.index(yval_val[0])
                          key_name = log_list[idx]
                          line_dict[key_name] = yval_val[1].strip()
  
                  # special case for ARCTUR PBS..
                  # Resource_List.nodes=1:ppn=1
                  if yval_val[0] == "Resource_List.nodes":
  
                      if len(yval_val[1].split(":")) > 1:
  
                          line_dict["nnodes"] = int(yval_val[1].split(":")[0])
  
                          if yval_val[1].split(":")[1] == "ppn":
                              line_dict["ncpus"] = int(yval_val[2]) * int(yval_val[1].split(":")[0])
                      else:
                          line_dict["ncpus"] = int(yval_val[1])
  
              i_job = IngestedJob()
  
              # print 'i_job.time_created ', line_dict['time_created']
              # print 'i_job.time_queued  ', line_dict['time_queued']
              # print 'i_job.time_eligible', line_dict['time_eligible']
              # print 'i_job.time_end     ', line_dict['time_end']
              # print 'i_job.time_start   ', line_dict['time_start']
              # print int(line_dict['ncpus'])
              # print line_dict['time_created']
              # print type( line_dict['time_created'] )
              # print any(c.isalpha() for c in line_dict['time_created'])
  
              # created  time
              if any([c.isalpha() for c in line_dict['time_created']]):
                  i_job.time_created = -1
              else:
                  i_job.time_created = int(line_dict['time_created'])
  
              # queue time
              if any([c.isalpha() for c in line_dict['time_queued']]):
                  i_job.time_queued = -1
              else:
                  i_job.time_queued = int(line_dict['time_queued'])
  
              # eligible time
              if any([c.isalpha() for c in line_dict['time_eligible']]):
                  i_job.time_eligible = -1
              else:
                  i_job.time_eligible = int(line_dict['time_eligible'])
  
              # end time
              if any([c.isalpha() for c in line_dict['time_end']]):
                  i_job.time_end = -1
              else:
                  i_job.time_end = int(line_dict['time_end'])
  
              # start time
              if any([c.isalpha() for c in line_dict['time_start']]):
                  i_job.time_start = -1
              else:
                  i_job.time_start = int(line_dict['time_start'])
  
              # average memory
              if any([c.isalpha() for c in line_dict['memory_kb'][:-2]]):
                  i_job.memory_kb = -1
              else:
                  i_job.memory_kb = int(line_dict['memory_kb'][:-2])
  
              if 'ncpus' in line_dict:
                  i_job.ncpus = int(line_dict['ncpus'])
              else:
                  i_job.ncpus = -1
  
              if 'nnodes' in line_dict:
                  i_job.nnodes = int(line_dict['nnodes'])
              else:
                  i_job.nnodes = -1
  
              # i_job.cpu_percent = float(line_dict['cpu_percent'].replace(":", ""))
              i_job.group = str(line_dict['group'])
              i_job.jobname = str(line_dict['jobname'])
              i_job.user = str(line_dict['user'])
              i_job.queue_type = str(line_dict['queue'])
              i_job.cmd_str = None  # command line string not available
  
              pbs_jobs.append(i_job)
  
              cce += 1
          cc += 1
  
      # remove invalid entries
      pbs_jobs[:] = [job for job in pbs_jobs if job.time_start != -1]
      pbs_jobs[:] = [job for job in pbs_jobs if job.time_end != -1]
      pbs_jobs[:] = [job for job in pbs_jobs if job.time_end >= job.time_start]
      pbs_jobs[:] = [job for job in pbs_jobs if job.time_queued != -1]
      pbs_jobs[:] = [job for job in pbs_jobs if job.time_start >= job.time_queued]
      pbs_jobs[:] = [job for job in pbs_jobs if job.ncpus > 0]
      pbs_jobs[:] = [job for job in pbs_jobs if job.nnodes > 0]
  
      for (ii, i_job) in enumerate(pbs_jobs):
          i_job.idx_in_log = ii
  
      pbs_jobs.sort(key=lambda x: x.time_start, reverse=False)
  
      # times relative to start of log
      min_start_time = min([i_job.time_start for i_job in pbs_jobs])
      for i_job in pbs_jobs:
          i_job.runtime = float(i_job.time_end) - float(i_job.time_start)
          i_job.time_start_0 = i_job.time_start - min_start_time
          i_job.time_in_queue = i_job.time_start - i_job.time_queued
  
      return pbs_jobs
  
  
  def read_accounting_logs(filename_in):
      """
      Reads ecmwf accounting logs
      :param filename_in:
      :return:
      """
  
      accounting_jobs = []
  
      # ['"hpc"',
      #  '"jobid"',
      #  '"jobname"',
      #  '"jobstepid"',
      #  '"owner_uid"',
      #  '"owner_group"',
      #  '"submitter_uid"',
      #  '"submitter_group"',
      #  '"queue_time"',
      #  '"start_time"',
      #  '"end_time"',
      #  '"no_nodes"',
      #  '"no_cpus"',
      #  '"class"',
      #  '"account"',
      #  '"usage"',
      #  '"sbu"',
      #  '"step_usertime"',
      #  '"stdin"',
      #  '"stdout"',
      #  '"stderr"',
      #  '"job_name"']
  
      with open(filename_in, 'rb') as csvfile:
  
          csv_dict = csv.DictReader(csvfile, delimiter=';', quotechar='"')
  
          for line_dict in csv_dict:
  
              i_job = IngestedJob()
  
              try:
                  i_job.time_queued = (datetime.strptime(line_dict['queue_time'], '%Y-%m-%d %H:%M:%S') -
                                       datetime(1970, 1, 1)).total_seconds()
              except ValueError:
                  i_job.time_queued = (datetime.strptime(line_dict['queue_time'], '%Y-%m-%d %H:%M:%S.%f') -
                                       datetime(1970, 1, 1)).total_seconds()
  
              try:
                  i_job.time_end = (datetime.strptime(line_dict['end_time'], '%Y-%m-%d %H:%M:%S.%f') -
                                    datetime(1970, 1, 1)).total_seconds()
              except ValueError:
                  i_job.time_end = (datetime.strptime(line_dict['end_time'], '%Y-%m-%d %H:%M:%S') -
                                    datetime(1970, 1, 1)).total_seconds()
  
              try:
                  i_job.time_start = (datetime.strptime(line_dict['start_time'], '%Y-%m-%d %H:%M:%S.%f') -
                                      datetime(1970, 1, 1)).total_seconds()
              except ValueError:
                  i_job.time_start = (datetime.strptime(line_dict['start_time'], '%Y-%m-%d %H:%M:%S') -
                                      datetime(1970, 1, 1)).total_seconds()
  
              if 'no_cpus' in line_dict:
                  i_job.ncpus = int(line_dict['no_cpus'])
              else:
                  i_job.ncpus = -1
  
              if 'no_nodes' in line_dict:
                  i_job.nnodes = int(line_dict['no_nodes'])
              else:
                  i_job.nnodes = -1
  
              if 'stdout' in line_dict:
                  i_job.stdout = line_dict['stdout']
              else:
                  i_job.stdout = []
  
              # i_job.cpu_percent = float(line_dict['cpu_percent'].replace(":", ""))
              i_job.group = str(line_dict['owner_group'])
              i_job.jobname = str(line_dict['jobname'])
              i_job.user = str(line_dict['owner_uid'])
              i_job.queue_type = str(line_dict['class'])
              i_job.cmd_str = None  # command line string not available
  
              # info not available
              i_job.time_created = -1
              i_job.time_eligible = -1
              i_job.memory_kb = -1
  
              accounting_jobs.append(i_job)
  
      # remove invalid entries
      accounting_jobs[:] = [job for job in accounting_jobs if job.time_start != -1]
      accounting_jobs[:] = [job for job in accounting_jobs if job.time_end != -1]
      accounting_jobs[:] = [job for job in accounting_jobs if job.time_end >= job.time_start]
      accounting_jobs[:] = [job for job in accounting_jobs if job.time_queued != -1]
      accounting_jobs[:] = [job for job in accounting_jobs if job.time_start >= job.time_queued]
      accounting_jobs[:] = [job for job in accounting_jobs if job.ncpus > 0]
      accounting_jobs[:] = [job for job in accounting_jobs if job.nnodes > 0]
  
      # store the original idx of each job..
      for (ii, i_job) in enumerate(accounting_jobs):
          i_job.idx_in_log = ii
  
      accounting_jobs.sort(key=lambda x: x.time_start, reverse=False)
  
      # times relative to start of log
      min_start_time = min([i_job.time_start for i_job in accounting_jobs])
      for i_job in accounting_jobs:
          # print type(i_job.time_queued), type(i_job.time_end), type(i_job.time_start)
          i_job.runtime = float(i_job.time_end) - float(i_job.time_start)
          i_job.time_start_0 = i_job.time_start - min_start_time
          i_job.time_in_queue = i_job.time_start - i_job.time_queued
  
      return accounting_jobs
  
  
  def read_epcc_csv_logs(filename_in):
  
      """ read CSV logs from EPCC.. """
  
      csv_jobs = []
  
      with open(filename_in, 'rb') as csvfile:
  
          csv_dict = csv.DictReader(csvfile, delimiter=',', quotechar='"')
  
          for line_dict in csv_dict:
  
              i_job = IngestedJob()
  
              # if isinstance(line_dict['ctime'], str):
              #     # i_job.time_queued = int(line_dict['ctime'])
              #     i_job.time_queued = int(line_dict['start']) + 999  # will be removed later..
              # else:
              #     print "line_dict['ctime']: ", line_dict['ctime']
              #     i_job.time_queued = int(line_dict['start']) + 999 #  will be removed later..
  
              try:
                  i_job.time_queued = int(line_dict['ctime'])
              except:
                  print(("I didn't recognize ctime {0} as a number".format(line_dict['ctime'])))
                  i_job.time_queued = -1
  
              try:
                  i_job.time_end = int(line_dict['end'])
              except:
                  print(("I didn't recognize end {0} as a number".format(line_dict['end'])))
                  i_job.time_end = -1
  
              try:
                  i_job.time_start = int(line_dict['start'])
              except:
                  print(("I didn't recognize start {0} as a number".format(line_dict['start'])))
                  i_job.time_start = -1
  
              try:
                  i_job.ncpus = int(line_dict['ncpus'])
              except:
                  print(("I didn't recognize start {0} as a number".format(line_dict['ncpus'])))
                  i_job.ncpus = -1
  
              try:
                  i_job.nnodes = int(line_dict['node_count'])
              except:
                  print(("I didn't recognize start {0} as a number".format(line_dict['node_count'])))
                  i_job.nnodes = -1
  
              # i_job.group = line_dict['group'].strip()
              i_job.group = ''
  
              if line_dict['jobname']:
                  i_job.jobname = line_dict['jobname'].strip()
              else:
                  i_job.jobname = ''
  
              if line_dict['jobname']:
                  i_job.user = line_dict['UserID'].strip()
              else:
                  i_job.user = ''
  
              if line_dict['jobname']:
                  i_job.queue_type = line_dict['queue'].strip()
              else:
                  i_job.queue_type = ''
  
              # info not available
              i_job.time_created = -1
              i_job.time_eligible = -1
              i_job.memory_kb = -1
              i_job.cmd_str = None  # command line string not available
  
              csv_jobs.append(i_job)
  
      # remove invalid entries
      csv_jobs[:] = [job for job in csv_jobs if job.time_start != -1]
      csv_jobs[:] = [job for job in csv_jobs if job.time_end != -1]
      csv_jobs[:] = [job for job in csv_jobs if job.time_end >= job.time_start]
      csv_jobs[:] = [job for job in csv_jobs if job.time_queued != -1]
      csv_jobs[:] = [job for job in csv_jobs if job.time_start >= job.time_queued]
      csv_jobs[:] = [job for job in csv_jobs if job.ncpus > 0]
      csv_jobs[:] = [job for job in csv_jobs if job.nnodes > 0]
  
      # store the original idx of each job..
      for (ii, i_job) in enumerate(csv_jobs):
          i_job.idx_in_log = ii
  
      csv_jobs.sort(key=lambda x: x.time_start, reverse=False)
  
      # times relative to start of log
      min_start_time = min([i_job.time_start for i_job in csv_jobs])
      for i_job in csv_jobs:
          # print type(i_job.time_queued), type(i_job.time_end), type(i_job.time_start)
          i_job.runtime = float(i_job.time_end) - float(i_job.time_start)
          i_job.time_start_0 = i_job.time_start - min_start_time
          i_job.time_in_queue = i_job.time_start - i_job.time_queued
  
      return csv_jobs
  
  
  class PBSDataSet(IngestedDataSet):
  
      def __init__(self, joblist, *args, **kwargs):
  
          super(PBSDataSet, self).__init__(joblist, '.', {'cache':False})
  
          # The created times are all in seconds since an arbitrary reference, so we want to get
          # them relative to a zero-time
          created_time_list = [j.time_created for j in self.joblist if j.time_created >= 0]
          self.global_created_time = 0.0
          if created_time_list:
              self.global_created_time = min(created_time_list)
  
          start_time_list = [j.time_created for j in self.joblist if j.time_created >= 0]
          self.global_start_time = 0.0
          if start_time_list:
              self.global_start_time = min(start_time_list)
  
      def model_jobs(self):
          for job in self.joblist:
              # assert isinstance(job, IngestedJob)
              assert not job.timesignals
  
              # if job.time_created >= 0:
              #     submit_time = job.time_created - self.global_created_time
              # else:
              #     submit_time = job.time_start - self.global_start_time
  
              yield ModelJob(
                  job_name=job.jobname,
                  user_name=job.user,
                  cmd_str=job.cmd_str,
                  queue_name=job.queue_type,
                  time_queued=job.time_queued,
                  time_start=job.time_start,
                  duration=job.time_end-job.time_start,
                  ncpus=job.ncpus,
                  nnodes=job.nnodes,
                  stdout=job.stdout,
                  label=None,
              )
  
  
  def ingest_pbs_logs(path, cfg=None):
      """
      Read PBS logs into a dataset
      """
      if not os.path.exists(path):
          raise ConfigurationError("Specified path to ingest PBS profiles does not exist: {}".format(path))
  
      if not os.path.isfile(path):
          raise ConfigurationError("Specified path for PBS time_schedule is not a file")
  
      jobs = read_pbs_log(path)
  
      return PBSDataSet(jobs)
  
  
  def ingest_epcc_csv_logs(path, cfg=None):
      """
      Read PBS logs into a dataset
      """
      if not os.path.exists(path):
          raise ConfigurationError("Specified path to ingest CSV profiles does not exist: {}".format(path))
  
      if not os.path.isfile(path):
          raise ConfigurationError("Specified path for CSV time_schedule is not a file")
  
      jobs = read_epcc_csv_logs(path)
  
      return PBSDataSet(jobs)
  
  
  class AccountingDataSet(IngestedDataSet):
  
      def __init__(self, joblist, *args, **kwargs):
          super(AccountingDataSet, self).__init__(joblist, '.', {'cache':False})
  
          # The created times are all in seconds since an arbitrary reference, so we want to get
          # them relative to a zero-time
          self.global_start_time = min((j.time_start for j in self.joblist if j.time_start >= 0))
  
      def model_jobs(self):
          for job in self.joblist:
              assert isinstance(job, IngestedJob)
              assert not job.timesignals
  
              # yield ModelJob(
              #     time_start=job.time_start - self.global_start_time,
              #     duration=job.time_end-job.time_start,
              #     ncpus=job.ncpus,
              #     nnodes=job.nnodes,
              #     scheduler_timing=job.time_queued,
              #     stdout=job.stdout
              # )
  
              yield ModelJob(
                  job_name=job.jobname,
                  user_name=job.user,
                  cmd_str=job.cmd_str,
                  queue_name=job.queue_type,
                  time_queued=job.time_queued,
                  time_start=job.time_start,
                  duration=job.time_end - job.time_start,
                  ncpus=job.ncpus,
                  nnodes=job.nnodes,
                  scheduler_timing=job.time_queued,
                  stdout=job.stdout,
                  label=None,
              )
  
  
  def ingest_accounting_logs(path, cfg=None):
      """
      Read PBS logs into a dataset
      """
      if not os.path.exists(path):
          raise ConfigurationError("Specified path to ingest accounting profiles does not exist: {}".format(path))
  
      if not os.path.isfile(path):
          raise ConfigurationError("Specified path for accounting log is not a file")
  
      jobs = read_accounting_logs(path)
  
      return PBSDataSet(jobs)
  
  