code: """
  #Trains a ResNet on the CIFAR10 dataset.
  
  """
  
  from __future__ import print_function
  import keras
  from keras.layers import Dense, Conv2D, BatchNormalization, Activation
  from keras.layers import AveragePooling2D, Input, Flatten
  from keras.optimizers import Adam
  from keras.callbacks import ModelCheckpoint, LearningRateScheduler
  from keras.callbacks import ReduceLROnPlateau, TensorBoard
  from keras.preprocessing.image import ImageDataGenerator
  from keras.regularizers import l2
  from keras import backend as K
  from keras.models import Model
  from keras.datasets import cifar10
  from keras.applications.vgg16 import VGG16
  from keras.applications.vgg19 import VGG19
  from keras import models, layers, optimizers
  from datetime import datetime
  import tensorflow as tf
  import numpy as np
  import os
  import pdb
  import sys
  import argparse
  import time
  import signal
  import glob
  import json
  import send_signal
  
  parser = argparse.ArgumentParser(description='Tensorflow Cifar10 Training')
  parser.add_argument('--tc', metavar='TESTCASE', type=str, help='specific testcase name')
  parser.add_argument('--resume', dest='resume', action='store_true', help='if True, resume training from a checkpoint')
  parser.add_argument('--gpu_num', metavar='GPU_NUMBER', type=str, help='select which gpu to use')
  parser.add_argument('--node', metavar='HOST_NODE', type=str, help='node of the host (scheduler)')
  parser.set_defaults(resume=False)
  args = parser.parse_args()
  
  os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID"
  os.environ["CUDA_VISIBLE_DEVICES"]=args.gpu_num
  
  # Training parameters
  batch_size = 256
  args_lr = 0.001
  args_model = 'vgg16'
  epoch_begin_time = 0
  
  job_name = sys.argv[0].split('.')[0]
  save_files = '/scratch/li.baol/checkpoint_final4_new2/' + job_name + '*'
  
  total_epochs = 6
  starting_epoch = 0
  
  # first step is to update the PID
  pid = os.getpid()
  message = job_name + ' pid ' + str(pid) # 'job50 pid 3333'
  send_signal.send(args.node, 10002, message)
  
  if args.resume:
      save_file = glob.glob(save_files)[0]
  #    epochs = int(save_file.split('/')[4].split('_')[1].split('.')[0])
      starting_epoch = int(save_file.split('/')[4].split('.')[0].split('_')[-1])
  
  data_augmentation = True
  num_classes = 10
  
  # Subtracting pixel mean improves accuracy
  subtract_pixel_mean = True
  
  n = 3
  
  # Model name, depth and version
  model_type = args.tc #'P100_resnet50_he_256_1'
  
  # Load the CIFAR10 data.
  (x_train, y_train), (x_test, y_test) = cifar10.load_data()
  
  # Normalize data.
  x_train = x_train.astype('float32') / 255
  x_test = x_test.astype('float32') / 255
  
  # If subtract pixel mean is enabled
  if subtract_pixel_mean:
      x_train_mean = np.mean(x_train, axis=0)
      x_train -= x_train_mean
      x_test -= x_train_mean
  
  print('x_train shape:', x_train.shape)
  print(x_train.shape[0], 'train samples')
  print(x_test.shape[0], 'test samples')
  print('y_train shape:', y_train.shape)
  
  # Convert class vectors to binary class matrices.
  y_train = keras.utils.to_categorical(y_train, num_classes)
  y_test = keras.utils.to_categorical(y_test, num_classes)
  
  if args.resume:
      print('resume from checkpoint')
      message = job_name + ' b_end'
      send_signal.send(args.node, 10002, message)
      model = keras.models.load_model(save_file)
      message = job_name + ' c_end'
      send_signal.send(args.node, 10002, message)
  else:
      print('train from start')
      model = models.Sequential()
      
      if '16' in args_model:
          base_model = VGG16(weights=None, include_top=False, input_shape=(32, 32, 3), pooling=None)
      elif '19' in args_model:
          base_model = VGG19(weights=None, include_top=False, input_shape=(32, 32, 3), pooling=None)
      
      #base_model.summary()
      
      #pdb.set_trace()
      
      model.add(base_model)
      model.add(layers.Flatten())
      model.add(layers.BatchNormalization())
      model.add(layers.Dense(128, activation='relu'))#, kernel_initializer='he_uniform'))
      #model.add(layers.Dropout(0.2))
      model.add(layers.BatchNormalization())
      model.add(layers.Dense(64, activation='relu'))#, kernel_initializer='he_uniform'))
      #model.add(layers.Dropout(0.2))
      model.add(layers.BatchNormalization())
      model.add(layers.Dense(10, activation='softmax'))#, kernel_initializer='he_uniform'))
      
      model.compile(loss='categorical_crossentropy',
                    optimizer=Adam(lr=args_lr),
                    metrics=['accuracy'])
      
      #model.summary()
      print(model_type)
  
  #pdb.set_trace()
  
  current_epoch = 0
  
  ################### connects interrupt signal to the process #####################
  
  def terminateProcess(signalNumber, frame):
      # first record the wasted epoch time
      global epoch_begin_time
      if epoch_begin_time == 0:
          epoch_waste_time = 0
      else:
          epoch_waste_time = int(time.time() - epoch_begin_time)
  
      message = job_name + ' waste ' + str(epoch_waste_time) # 'job50 waste 100'
      if epoch_waste_time > 0:
          send_signal.send(args.node, 10002, message)
  
      print('checkpointing the model triggered by kill -15 signal')
      # delete whatever checkpoint that already exists
      for f in glob.glob(save_files):
          os.remove(f)
      model.save('/scratch/li.baol/checkpoint_final4_new2/' + job_name + '_' + str(current_epoch) + '.h5')
      print ('(SIGTERM) terminating the process')
  
      message = job_name + ' checkpoint'
      send_signal.send(args.node, 10002, message)
  
      sys.exit()
  
  signal.signal(signal.SIGTERM, terminateProcess)
  
  #################################################################################
  
  logdir = '/scratch/li.baol/tsrbrd_log/job_runs/' + model_type + '/' + job_name
  
  tensorboard_callback = TensorBoard(log_dir=logdir)#, update_freq='batch')
  
  first_epoch_start = 0
  
  class PrintEpoch(keras.callbacks.Callback):
      def on_epoch_begin(self, epoch, logs=None):
          global current_epoch, first_epoch_start
          #remaining_epochs = epochs - epoch
          current_epoch = epoch
          print('current epoch ' + str(current_epoch))
          global epoch_begin_time
          epoch_begin_time = time.time()
          if epoch == starting_epoch and args.resume:
              first_epoch_start = time.time()
              message = job_name + ' d_end'
              send_signal.send(args.node, 10002, message)
          elif epoch == starting_epoch:
              first_epoch_start = time.time()           
          if epoch == starting_epoch:
              # send signal to indicate checkpoint is qualified
              message = job_name + ' ckpt_qual'
              send_signal.send(args.node, 10002, message)
  
  
      def on_epoch_end(self, epoch, logs=None):
          if epoch == starting_epoch:
              first_epoch_time = int(time.time() - first_epoch_start)
              message = job_name + ' 1st_epoch ' + str(first_epoch_time)
              send_signal.send(args.node, 10002, message)
          progress = round((epoch+1) / round(total_epochs/2), 2)
          message = job_name + ' completion ' + str(progress)
          send_signal.send(args.node, 10002, message)
  
  my_callback = PrintEpoch()
  
  callbacks = [tensorboard_callback, my_callback]
   #[checkpoint, lr_reducer, lr_scheduler, tensorboard_callback]
  
  # Run training
  
  model.fit(x_train, y_train,
            batch_size=batch_size,
            epochs=round(total_epochs/2),
            validation_data=(x_test, y_test),
            shuffle=True,
            callbacks=callbacks,
            initial_epoch=starting_epoch,
            verbose=1
            )
  
  # Score trained model.
  scores = model.evaluate(x_test, y_test, verbose=1)
  print('Test loss:', scores[0])
  print('Test accuracy:', scores[1])
  
  # send signal to indicate job has finished
  message = job_name + ' finish'
  send_signal.send(args.node, 10002, message)
  
   """
  #Trains a ResNet on the CIFAR10 dataset.
  
  """
  
  from __future__ import print_function
  import keras
  from keras.layers import Dense, Conv2D, BatchNormalization, Activation
  from keras.layers import AveragePooling2D, Input, Flatten
  from keras.optimizers import Adam
  from keras.callbacks import ModelCheckpoint, LearningRateScheduler
  from keras.callbacks import ReduceLROnPlateau, TensorBoard
  from keras.preprocessing.image import ImageDataGenerator
  from keras.regularizers import l2
  from keras import backend as K
  from keras.models import Model
  from keras.datasets import cifar10
  from keras.applications.resnet import ResNet50, ResNet101, ResNet152
  from keras import models, layers, optimizers
  from datetime import datetime
  import tensorflow as tf
  import numpy as np
  import os
  import pdb
  import sys
  import argparse
  import time
  import signal
  import glob
  import json
  import send_signal
  
  parser = argparse.ArgumentParser(description='Tensorflow Cifar10 Training')
  parser.add_argument('--tc', metavar='TESTCASE', type=str, help='specific testcase name')
  parser.add_argument('--resume', dest='resume', action='store_true', help='if True, resume training from a checkpoint')
  parser.add_argument('--gpu_num', metavar='GPU_NUMBER', type=str, help='select which gpu to use')
  parser.add_argument('--node', metavar='HOST_NODE', type=str, help='node of the host (scheduler)')
  parser.set_defaults(resume=False)
  args = parser.parse_args()
  
  os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID"
  os.environ["CUDA_VISIBLE_DEVICES"]=args.gpu_num
  
  # Training parameters
  batch_size = 32
  args_lr = 0.0014
  args_model = 'resnet101'
  
  epoch_begin_time = 0
  
  job_name = sys.argv[0].split('.')[0]
  save_files = '/scratch/li.baol/checkpoint_final4/' + job_name + '*'
  
  total_epochs = 36
  starting_epoch = 0
  
  # first step is to update the PID
  pid = os.getpid()
  message = job_name + ' pid ' + str(pid) # 'job50 pid 3333'
  send_signal.send(args.node, 10002, message)
  
  if args.resume:
      save_file = glob.glob(save_files)[0]
  #    epochs = int(save_file.split('/')[4].split('_')[1].split('.')[0])
      starting_epoch = int(save_file.split('/')[4].split('.')[0].split('_')[-1])
  
  data_augmentation = True
  num_classes = 10
  
  # Subtracting pixel mean improves accuracy
  subtract_pixel_mean = True
  
  n = 3
  
  # Model name, depth and version
  model_type = args.tc #'P100_resnet50_he_256_1'
  
  # Load the CIFAR10 data.
  (x_train, y_train), (x_test, y_test) = cifar10.load_data()
  
  # Normalize data.
  x_train = x_train.astype('float32') / 255
  x_test = x_test.astype('float32') / 255
  
  # If subtract pixel mean is enabled
  if subtract_pixel_mean:
      x_train_mean = np.mean(x_train, axis=0)
      x_train -= x_train_mean
      x_test -= x_train_mean
  
  print('x_train shape:', x_train.shape)
  print(x_train.shape[0], 'train samples')
  print(x_test.shape[0], 'test samples')
  print('y_train shape:', y_train.shape)
  
  # Convert class vectors to binary class matrices.
  y_train = keras.utils.to_categorical(y_train, num_classes)
  y_test = keras.utils.to_categorical(y_test, num_classes)
  
  if args.resume:
      print('resume from checkpoint')
      message = job_name + ' b_end'
      send_signal.send(args.node, 10002, message)
      model = keras.models.load_model(save_file)
      message = job_name + ' c_end'
      send_signal.send(args.node, 10002, message)
  else:
      print('train from start')
      model = models.Sequential()
      
      if '50' in args_model:
          base_model = ResNet50(weights=None, include_top=False, input_shape=(32, 32, 3), pooling=None)
      elif '101' in args_model:
          base_model = ResNet101(weights=None, include_top=False, input_shape=(32, 32, 3), pooling=None)
      elif '152' in args_model:
          base_model = ResNet152(weights=None, include_top=False, input_shape=(32, 32, 3), pooling=None)
      
      #base_model.summary()
      
      #pdb.set_trace()
      
      #model.add(layers.UpSampling2D((2,2)))
      #model.add(layers.UpSampling2D((2,2)))
      #model.add(layers.UpSampling2D((2,2)))
      model.add(base_model)
      model.add(layers.Flatten())
      #model.add(layers.BatchNormalization())
      #model.add(layers.Dense(128, activation='relu'))
      #model.add(layers.Dropout(0.5))
      #model.add(layers.BatchNormalization())
      #model.add(layers.Dense(64, activation='relu'))
      #model.add(layers.Dropout(0.5))
      #model.add(layers.BatchNormalization())
      model.add(layers.Dense(10, activation='softmax'))#, kernel_initializer='he_uniform'))
      
      model.compile(loss='categorical_crossentropy',
                    optimizer=Adam(lr=args_lr),
                    metrics=['accuracy'])
      
      #model.summary()
      print(model_type)
  
  #pdb.set_trace()
  
  current_epoch = 0
  
  ################### connects interrupt signal to the process #####################
  
  def terminateProcess(signalNumber, frame):
      # first record the wasted epoch time
      global epoch_begin_time
      if epoch_begin_time == 0:
          epoch_waste_time = 0
      else:
          epoch_waste_time = int(time.time() - epoch_begin_time)
  
      message = job_name + ' waste ' + str(epoch_waste_time) # 'job50 waste 100'
      if epoch_waste_time > 0:
          send_signal.send(args.node, 10002, message)
  
      print('checkpointing the model triggered by kill -15 signal')
      # delete whatever checkpoint that already exists
      for f in glob.glob(save_files):
          os.remove(f)
      model.save('/scratch/li.baol/checkpoint_final4/' + job_name + '_' + str(current_epoch) + '.h5')
      print ('(SIGTERM) terminating the process')
  
      message = job_name + ' checkpoint'
      send_signal.send(args.node, 10002, message)
  
      sys.exit()
  
  signal.signal(signal.SIGTERM, terminateProcess)
  
  #################################################################################
  
  logdir = '/scratch/li.baol/tsrbrd_log/job_runs/' + model_type + '/' + job_name
  
  tensorboard_callback = TensorBoard(log_dir=logdir)#, update_freq='batch')
  
  first_epoch_start = 0
  
  class PrintEpoch(keras.callbacks.Callback):
      def on_epoch_begin(self, epoch, logs=None):
          global current_epoch, first_epoch_start
          #remaining_epochs = epochs - epoch
          current_epoch = epoch
          print('current epoch ' + str(current_epoch))
          global epoch_begin_time
          epoch_begin_time = time.time()
          if epoch == starting_epoch and args.resume:
              first_epoch_start = time.time()
              message = job_name + ' d_end'
              send_signal.send(args.node, 10002, message)
          elif epoch == starting_epoch:
              first_epoch_start = time.time()           
          if epoch == starting_epoch:
              # send signal to indicate checkpoint is qualified
              message = job_name + ' ckpt_qual'
              send_signal.send(args.node, 10002, message)
  
  
      def on_epoch_end(self, epoch, logs=None):
          if epoch == starting_epoch:
              first_epoch_time = int(time.time() - first_epoch_start)
              message = job_name + ' 1st_epoch ' + str(first_epoch_time)
              send_signal.send(args.node, 10002, message)
          progress = round((epoch+1) / round(total_epochs/2), 2)
          message = job_name + ' completion ' + str(progress)
          send_signal.send(args.node, 10002, message)
  
  my_callback = PrintEpoch()
  
  callbacks = [tensorboard_callback, my_callback]
   #[checkpoint, lr_reducer, lr_scheduler, tensorboard_callback]
  
  # Run training
  
  model.fit(x_train, y_train,
            batch_size=batch_size,
            epochs=round(total_epochs/2),
            validation_data=(x_test, y_test),
            shuffle=True,
            callbacks=callbacks,
            initial_epoch=starting_epoch,
            verbose=1
            )
  
  # Score trained model.
  scores = model.evaluate(x_test, y_test, verbose=1)
  print('Test loss:', scores[0])
  print('Test accuracy:', scores[1])
  
  # send signal to indicate job has finished
  message = job_name + ' finish'
  send_signal.send(args.node, 10002, message)
   """
  #Trains a ResNet on the CIFAR10 dataset.
  
  """
  
  from __future__ import print_function
  import keras
  from keras.layers import Dense, Conv2D, BatchNormalization, Activation
  from keras.layers import AveragePooling2D, Input, Flatten
  from keras.optimizers import Adam
  from keras.callbacks import ModelCheckpoint, LearningRateScheduler
  from keras.callbacks import ReduceLROnPlateau, TensorBoard
  from keras.preprocessing.image import ImageDataGenerator
  from keras.regularizers import l2
  from keras import backend as K
  from keras.models import Model
  from keras.datasets import cifar10
  from keras.applications.vgg16 import VGG16
  from keras.applications.vgg19 import VGG19
  from keras import models, layers, optimizers
  from datetime import datetime
  import tensorflow as tf
  import numpy as np
  import os
  import pdb
  import sys
  import argparse
  import time
  import signal
  import glob
  import json
  import send_signal
  
  load_start = time.time()
  
  parser = argparse.ArgumentParser(description='Tensorflow Cifar10 Training')
  parser.add_argument('--tc', metavar='TESTCASE', type=str, help='specific testcase name')
  parser.add_argument('--resume', dest='resume', action='store_true', help='if True, resume training from a checkpoint')
  parser.add_argument('--gpu_num', metavar='GPU_NUMBER', type=str, help='select which gpu to use')
  parser.add_argument('--node', metavar='HOST_NODE', type=str, help='node of the host (scheduler)')
  parser.set_defaults(resume=False)
  args = parser.parse_args()
  
  os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID"
  os.environ["CUDA_VISIBLE_DEVICES"]=args.gpu_num
  
  # Training parameters
  batch_size = 256
  args_lr = 0.005
  args_model = 'vgg16'
  
  epoch_begin_time = 0
  
  job_name = sys.argv[0].split('.')[0]
  save_files = '/scratch/li.baol/checkpoint_test/' + job_name + '*'
  
  total_epochs = 9
  starting_epoch = 0
  
  if args.resume:
      save_file = glob.glob(save_files)[0]
  #    epochs = int(save_file.split('/')[4].split('_')[1].split('.')[0])
      starting_epoch = int(save_file.split('/')[4].split('.')[0].split('_')[-1])
  
  data_augmentation = True
  num_classes = 10
  
  # Subtracting pixel mean improves accuracy
  subtract_pixel_mean = True
  
  n = 3
  
  # Model name, depth and version
  model_type = args.tc #'P100_resnet50_he_256_1'
  
  # Load the CIFAR10 data.
  (x_train, y_train), (x_test, y_test) = cifar10.load_data()
  
  # Normalize data.
  x_train = x_train.astype('float32') / 255
  x_test = x_test.astype('float32') / 255
  
  # If subtract pixel mean is enabled
  if subtract_pixel_mean:
      x_train_mean = np.mean(x_train, axis=0)
      x_train -= x_train_mean
      x_test -= x_train_mean
  
  print('x_train shape:', x_train.shape)
  print(x_train.shape[0], 'train samples')
  print(x_test.shape[0], 'test samples')
  print('y_train shape:', y_train.shape)
  
  # Convert class vectors to binary class matrices.
  y_train = keras.utils.to_categorical(y_train, num_classes)
  y_test = keras.utils.to_categorical(y_test, num_classes)
  
  if args.resume:
      print('resume from checkpoint')
      model = keras.models.load_model(save_file)
  else:
      print('train from start')
      model = models.Sequential()
      
      if '16' in args_model:
          base_model = VGG16(weights=None, include_top=False, input_shape=(32, 32, 3), pooling=None)
      elif '19' in args_model:
          base_model = VGG19(weights=None, include_top=False, input_shape=(32, 32, 3), pooling=None)
      
      #base_model.summary()
      
      #pdb.set_trace()
      
      model.add(base_model)
      model.add(layers.Flatten())
      model.add(layers.BatchNormalization())
      model.add(layers.Dense(128, activation='relu'))#, kernel_initializer='he_uniform'))
      #model.add(layers.Dropout(0.2))
      model.add(layers.BatchNormalization())
      model.add(layers.Dense(64, activation='relu'))#, kernel_initializer='he_uniform'))
      #model.add(layers.Dropout(0.2))
      model.add(layers.BatchNormalization())
      model.add(layers.Dense(10, activation='softmax'))#, kernel_initializer='he_uniform'))
      
      model.compile(loss='categorical_crossentropy',
                    optimizer=Adam(lr=args_lr),
                    metrics=['accuracy'])
      
      #model.summary()
      print(model_type)
  
  #pdb.set_trace()
  
  current_epoch = 0
  
  ################### connects interrupt signal to the process #####################
  
  def terminateProcess():
      save_start = time.time()
      # first record the wasted epoch time
      global epoch_begin_time
      if epoch_begin_time == 0:
          epoch_waste_time = 0
      else:
          epoch_waste_time = int(time.time() - epoch_begin_time)
  
      print('checkpointing the model triggered by kill -15 signal')
      # delete whatever checkpoint that already exists
      for f in glob.glob(save_files):
          os.remove(f)
      model.save('/scratch/li.baol/checkpoint_test/' + job_name + '_' + str(current_epoch) + '.h5')
      print ('(SIGTERM) terminating the process')
  
      save_time = int(time.time() - save_start)
      message = job_name + ' save ' + str(save_time)
      send_signal.send(args.node, 10002, message)
  
      sys.exit()
  
  signal.signal(signal.SIGTERM, terminateProcess)
  
  #################################################################################
  
  logdir = '/scratch/li.baol/tsrbrd_log/job_runs/' + model_type + '/' + job_name
  
  tensorboard_callback = TensorBoard(log_dir=logdir)#, update_freq='batch')
  
  class PrintEpoch(keras.callbacks.Callback):
      def on_epoch_begin(self, epoch, logs=None):
          global current_epoch 
          #remaining_epochs = epochs - epoch
          current_epoch = epoch
          print('current epoch ' + str(current_epoch))
          global epoch_begin_time
          epoch_begin_time = time.time()
  
  my_callback = PrintEpoch()
  
  callbacks = [tensorboard_callback, my_callback]
  
  load_time = int(time.time() - load_start)
  if args.resume:
      message = job_name + ' load ' + str(load_time)
      send_signal.send(args.node, 10002, message)
      # Score trained model.
      scores = model.evaluate(x_test, y_test, verbose=1)
      print('Test loss:', scores[0])
      print('Test accuracy:', scores[1])
      # send signal to indicate job has finished
      message = job_name + ' finish'
      send_signal.send(args.node, 10002, message)
      sys.exit()
  
  
  model.fit(x_train, y_train,
            batch_size=batch_size,
            epochs=1,
            validation_data=(x_test, y_test),
            shuffle=True,
            callbacks=callbacks,
            initial_epoch=starting_epoch,
            verbose=1
            )
  if not args.resume:
      terminateProcess()
   import pandas
  import pdb
  from datetime import datetime
  import matplotlib
  import numpy as np
  import matplotlib
  matplotlib.use('Agg')
  import matplotlib.pyplot as plt
  import glob
  import sys
  from matplotlib.ticker import MultipleLocator
  from scipy.stats import pearsonr, spearmanr
  from sklearn import neighbors
  from sklearn.metrics import mean_squared_error
  from math import sqrt
  from sklearn.model_selection import train_test_split
  from sklearn.linear_model import LinearRegression
  import json
  
  log_dir = '/scratch/li.baol/GPU_pwr_meas/tensorflow/round1/regression/pwr/*'
  dirs = glob.glob(log_dir)
  dirs.sort()
  # store everything in a dict
  all_pwr = {} # {densenet121_32:{K80:a, K100:b}...}
  
  for tc in dirs:
      test = tc.split('/')[6+1+1].split('.')[0]
      gpu = test.split('_')[0]
      model = test.replace(gpu + '_', '')
  
      # read tc.csv into a list
      data = pandas.read_csv(tc)
      pwr = np.asarray(data[data.columns[0]].tolist())
      
      if model in all_pwr:
          all_pwr[model][gpu] = pwr
      else:
          all_pwr[model] = {gpu: pwr}
  
  log_dir = '/scratch/li.baol/GPU_pwr_meas/tensorflow/round1/regression/util/*'
  dirs = glob.glob(log_dir)
  dirs.sort()
  # store everything in a dict
  all_util = {} # {densenet121_32:{K80:a, K100:b}...}
  
  for tc in dirs:
      test = tc.split('/')[6+1+1].split('.')[0]
      gpu = test.split('_')[0]
      model = test.replace(gpu + '_', '')
  
      # read tc.csv into a list
      data = pandas.read_csv(tc)
      util = np.asarray(data[data.columns[0]].tolist())
      
      if model in all_util:
          all_util[model][gpu] = util
      else:
          all_util[model] = {gpu: util}
  
  log_dir = '/scratch/li.baol/GPU_pwr_meas/tensorflow/round1/regression/mem_util/*'
  dirs = glob.glob(log_dir)
  dirs.sort()
  # store everything in a dict
  all_mem_util = {} # {densenet121_32:{K80:a, K100:b}...}
  
  for tc in dirs:
      test = tc.split('/')[6+1+1].split('.')[0]
      gpu = test.split('_')[0]
      model = test.replace(gpu + '_', '')
  
      # read tc.csv into a list
      data = pandas.read_csv(tc)
      mem_util = np.asarray(data[data.columns[0]].tolist())
      
      if model in all_mem_util:
          all_mem_util[model][gpu] = mem_util
      else:
          all_mem_util[model] = {gpu: mem_util}
  
  log_dir = '/scratch/li.baol/GPU_time_meas/tensorflow/round1/csv/*'
  dirs = glob.glob(log_dir)
  dirs.sort()
  # store everything in a dict
  all_time = {} # {densenet121_32:{K80:a, K100:b}...}
  
  for tc in dirs:
      test = tc.split('/')[6+1].split('.')[0]
      gpu = test.split('_')[0]
      model = test.replace(gpu + '_', '')
  
      # read tc.csv into a list
      data = pandas.read_csv(tc)
      time = np.asarray(data[data.columns[0]].tolist())
      
      if model in all_time:
          all_time[model][gpu] = time
      else:
          all_time[model] = {gpu: time}
  
  # Now plot V100 power save ratio (%) vs K80 power(W)
  
  x1_data = [] # power
  x2_data = [] # speed
  x3_data = [] # utilization
  x4_data = [] # mem util
  y_data = []
  
  for key in all_pwr:
  #    if ('mnasnet' not in key and 'mobilenet' not in key):
      for i in all_pwr[key]['K80'].tolist(): # power
          x1_data.append(i)
      for i in (1 / all_time[key]['K80']).tolist(): # speed
          x2_data.append(i)
      for i in (all_util[key]['K80']).tolist(): # utilization
          x3_data.append(i)
      for i in (all_mem_util[key]['K80']).tolist(): # mem util
          x4_data.append(i)
      for i in (all_time[key]['K80'] / all_time[key]['V100']).tolist(): # speed up  
          y_data.append(i)
  
  x1_norm = [(i - min(x1_data)) / (max(x1_data) - min(x1_data)) for i in x1_data]
  x2_norm = [(i - min(x2_data)) / (max(x2_data) - min(x2_data)) for i in x2_data]
  x3_norm = [(i - min(x3_data)) / (max(x3_data) - min(x3_data)) for i in x3_data]
  x4_norm = [(i - min(x4_data)) / (max(x4_data) - min(x4_data)) for i in x4_data]
  
  # create training data
  x_data = []
  for i in range(len(x1_norm)):
      x_data.append([x1_norm[i], x2_norm[i], x3_norm[i], x4_norm[i]])
  
  x_train, x_test, y_train, y_test = train_test_split(x_data, y_data, test_size=0.3)
  
  with open('x1_data.json', 'w') as outfile:
      json.dump(x1_data, outfile)
  with open('x2_data.json', 'w') as outfile:
      json.dump(x2_data, outfile)
  with open('x3_data.json', 'w') as outfile:
      json.dump(x3_data, outfile)
  with open('x4_data.json', 'w') as outfile:
      json.dump(x4_data, outfile)
  
  with open('y_data.json', 'w') as outfile:
      json.dump(y_data, outfile)
  #with open('x_data.json') as f:
  #    x_data = json.load(f)
  #with open('y_data.json') as f:
  #    y_data = json.load(f)
  #x_train, x_test, y_train, y_test = train_test_split(x_data, y_data, test_size=0.3)
  
  rmse_val = [] #to store rmse values for different k
  for K in range(20):
      K = K+1
      model = neighbors.KNeighborsRegressor(n_neighbors = K, weights='distance')
  
      model.fit(x_train, y_train)  #fit the model
      pred = model.predict(x_test) #make prediction on test set
  #    model.predict(np.array(x_test[0]).reshape((1, -1)))
      err = sqrt(mean_squared_error(y_test, pred)) #calculate rmse
      rmse_val.append(err) #store rmse values
      err_pct = abs(y_test-pred) / y_test * 100
      print('RMSE value for k= ' , K , 'is:', err)
      print('error (%) is', np.mean(err_pct))
  
  xx_data = []
  for i in range(len(x1_norm)):
      xx_data.append([x1_norm[i]])
  
  # now compare with liear regression
  x_train, x_test, y_train, y_test = train_test_split(xx_data, y_data, test_size=0.3)
  model2 = LinearRegression().fit(x_train, y_train)
  pred = model2.predict(x_test) #make prediction on test set
  err = sqrt(mean_squared_error(y_test,pred)) #calculate rmse
  print('RMSE value for linear regression is ', err)
  
  
  
   """
  #Trains a ResNet on the CIFAR10 dataset.
  
  """
  
  from __future__ import print_function
  import keras
  from keras.layers import Dense, Conv2D, BatchNormalization, Activation
  from keras.layers import AveragePooling2D, Input, Flatten
  from keras.optimizers import Adam
  from keras.callbacks import ModelCheckpoint, LearningRateScheduler
  from keras.callbacks import ReduceLROnPlateau, TensorBoard
  from keras.preprocessing.image import ImageDataGenerator
  from keras.regularizers import l2
  from keras import backend as K
  from keras.models import Model
  from keras.datasets import cifar10
  from keras.applications.mobilenet_v2 import MobileNetV2
  from keras import models, layers, optimizers
  from datetime import datetime
  import tensorflow as tf
  import numpy as np
  import os
  import pdb
  import sys
  import argparse
  import time
  import signal
  import glob
  import json
  import send_signal
  
  parser = argparse.ArgumentParser(description='Tensorflow Cifar10 Training')
  parser.add_argument('--tc', metavar='TESTCASE', type=str, help='specific testcase name')
  parser.add_argument('--resume', dest='resume', action='store_true', help='if True, resume training from a checkpoint')
  parser.add_argument('--gpu_num', metavar='GPU_NUMBER', type=str, help='select which gpu to use')
  parser.add_argument('--node', metavar='HOST_NODE', type=str, help='node of the host (scheduler)')
  parser.set_defaults(resume=False)
  args = parser.parse_args()
  
  os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID"
  os.environ["CUDA_VISIBLE_DEVICES"]=args.gpu_num
  
  # Training parameters
  batch_size = 256
  args_lr = 0.0005
  
  epoch_begin_time = 0
  
  job_name = sys.argv[0].split('.')[0]
  save_files = '/scratch/li.baol/checkpoint_final1/' + job_name + '*'
  
  total_epochs = 44
  starting_epoch = 0
  
  # first step is to update the PID
  pid_dict = {}
  with open('pid_lock.json', 'r') as fp:
      pid_dict = json.load(fp)
  pid_dict[job_name] = os.getpid()
  json_file = json.dumps(pid_dict)
  with open('pid_lock.json', 'w') as fp:
      fp.write(json_file) 
  os.rename('pid_lock.json', 'pid.json')
  
  if args.resume:
      save_file = glob.glob(save_files)[0]
  #    epochs = int(save_file.split('/')[4].split('_')[1].split('.')[0])
      starting_epoch = int(save_file.split('/')[4].split('.')[0].split('_')[-1])
  
  data_augmentation = True
  num_classes = 10
  
  # Subtracting pixel mean improves accuracy
  subtract_pixel_mean = True
  
  n = 3
  
  # Model name, depth and version
  model_type = args.tc #'P100_resnet50_he_256_1'
  
  # Load the CIFAR10 data.
  (x_train, y_train), (x_test, y_test) = cifar10.load_data()
  
  # Normalize data.
  x_train = x_train.astype('float32') / 255
  x_test = x_test.astype('float32') / 255
  
  # If subtract pixel mean is enabled
  if subtract_pixel_mean:
      x_train_mean = np.mean(x_train, axis=0)
      x_train -= x_train_mean
      x_test -= x_train_mean
  
  print('x_train shape:', x_train.shape)
  print(x_train.shape[0], 'train samples')
  print(x_test.shape[0], 'test samples')
  print('y_train shape:', y_train.shape)
  
  # Convert class vectors to binary class matrices.
  y_train = keras.utils.to_categorical(y_train, num_classes)
  y_test = keras.utils.to_categorical(y_test, num_classes)
  
  if args.resume:
      print('resume from checkpoint')
      model = keras.models.load_model(save_file)
  else:
      print('train from start')
      model = models.Sequential()
      
      base_model = MobileNetV2(weights=None, include_top=False, input_shape=(32, 32, 3), pooling=None)
      
      #base_model.summary()
      
      #pdb.set_trace()
      
      model.add(base_model)
      model.add(layers.Flatten())
      #model.add(layers.BatchNormalization())
      #model.add(layers.Dense(128, activation='relu'))
      #model.add(layers.Dropout(0.5))
      #model.add(layers.BatchNormalization())
      #model.add(layers.Dense(64, activation='relu'))
      #model.add(layers.Dropout(0.5))
      #model.add(layers.BatchNormalization())
      model.add(layers.Dense(10, activation='softmax'))
      
      model.compile(loss='categorical_crossentropy',
                    optimizer=Adam(lr=args_lr),
                    metrics=['accuracy'])
      
      #model.summary()
      print(model_type)
  
  #pdb.set_trace()
  
  current_epoch = 0
  
  ################### connects interrupt signal to the process #####################
  
  def terminateProcess(signalNumber, frame):
      # first record the wasted epoch time
      global epoch_begin_time
      if epoch_begin_time == 0:
          epoch_waste_time = 0
      else:
          epoch_waste_time = int(time.time() - epoch_begin_time)
  
      epoch_waste_dict = {}
      with open('epoch_waste.json', 'r') as fp:
          epoch_waste_dict = json.load(fp)
      epoch_waste_dict[job_name] += epoch_waste_time
      json_file3 = json.dumps(epoch_waste_dict)
      with open('epoch_waste.json', 'w') as fp:
          fp.write(json_file3)
  
      print('checkpointing the model triggered by kill -15 signal')
      # delete whatever checkpoint that already exists
      for f in glob.glob(save_files):
          os.remove(f)
      model.save('/scratch/li.baol/checkpoint_final1/' + job_name + '_' + str(current_epoch) + '.h5')
      print ('(SIGTERM) terminating the process')
  
      checkpoint_dict = {}
      with open('checkpoint.json', 'r') as fp:
          checkpoint_dict = json.load(fp)
      checkpoint_dict[job_name] = 1
      json_file3 = json.dumps(checkpoint_dict)
      with open('checkpoint.json', 'w') as fp:
          fp.write(json_file3)
  
      sys.exit()
  
  signal.signal(signal.SIGTERM, terminateProcess)
  
  #################################################################################
  
  logdir = '/scratch/li.baol/tsrbrd_log/job_runs/' + model_type + '/' + job_name
  
  tensorboard_callback = TensorBoard(log_dir=logdir)#, update_freq='batch')
  
  class PrintEpoch(keras.callbacks.Callback):
      def on_epoch_begin(self, epoch, logs=None):
          global current_epoch 
          #remaining_epochs = epochs - epoch
          current_epoch = epoch
          print('current epoch ' + str(current_epoch))
          global epoch_begin_time
          epoch_begin_time = time.time()
  
  my_callback = PrintEpoch()
  
  callbacks = [tensorboard_callback, my_callback]
   #[checkpoint, lr_reducer, lr_scheduler, tensorboard_callback]
  
  # Run training
  
  # send signal to indicate checkpoint is qualified
  message = job_name + ' ckpt_qual'
  send_signal.send(args.node, 10002, message)
  
  model.fit(x_train, y_train,
            batch_size=batch_size,
            epochs=round(total_epochs/2),
            validation_data=(x_test, y_test),
            shuffle=True,
            callbacks=callbacks,
            initial_epoch=starting_epoch,
            verbose=1
            )
  
  # Score trained model.
  scores = model.evaluate(x_test, y_test, verbose=1)
  print('Test loss:', scores[0])
  print('Test accuracy:', scores[1])
  
  # send signal to indicate job has finished
  message = job_name + ' finish'
  send_signal.send(args.node, 10002, message)
   """
  #Trains a ResNet on the CIFAR10 dataset.
  
  """
  
  from __future__ import print_function
  import keras
  from keras.layers import Dense, Conv2D, BatchNormalization, Activation
  from keras.layers import AveragePooling2D, Input, Flatten
  from keras.optimizers import Adam
  from keras.callbacks import ModelCheckpoint, LearningRateScheduler
  from keras.callbacks import ReduceLROnPlateau, TensorBoard
  from keras.preprocessing.image import ImageDataGenerator
  from keras.regularizers import l2
  from keras import backend as K
  from keras.models import Model
  from keras.datasets import cifar10
  from keras.applications.resnet import ResNet50, ResNet101, ResNet152
  from keras import models, layers, optimizers
  from datetime import datetime
  import tensorflow as tf
  import numpy as np
  import os
  import pdb
  import sys
  import argparse
  import time
  import signal
  import glob
  import json
  import send_signal
  
  parser = argparse.ArgumentParser(description='Tensorflow Cifar10 Training')
  parser.add_argument('--tc', metavar='TESTCASE', type=str, help='specific testcase name')
  parser.add_argument('--resume', dest='resume', action='store_true', help='if True, resume training from a checkpoint')
  parser.add_argument('--gpu_num', metavar='GPU_NUMBER', type=str, help='select which gpu to use')
  parser.add_argument('--node', metavar='HOST_NODE', type=str, help='node of the host (scheduler)')
  parser.set_defaults(resume=False)
  args = parser.parse_args()
  
  os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID"
  os.environ["CUDA_VISIBLE_DEVICES"]=args.gpu_num
  
  # Training parameters
  batch_size = 32
  args_lr = 0.0014
  args_model = 'resnet101'
  
  epoch_begin_time = 0
  
  job_name = sys.argv[0].split('.')[0]
  save_files = '/scratch/li.baol/checkpoint_final4/' + job_name + '*'
  
  total_epochs = 134
  starting_epoch = 0
  
  # first step is to update the PID
  pid = os.getpid()
  message = job_name + ' pid ' + str(pid) # 'job50 pid 3333'
  send_signal.send(args.node, 10002, message)
  
  if args.resume:
      save_file = glob.glob(save_files)[0]
  #    epochs = int(save_file.split('/')[4].split('_')[1].split('.')[0])
      starting_epoch = int(save_file.split('/')[4].split('.')[0].split('_')[-1])
  
  data_augmentation = True
  num_classes = 10
  
  # Subtracting pixel mean improves accuracy
  subtract_pixel_mean = True
  
  n = 3
  
  # Model name, depth and version
  model_type = args.tc #'P100_resnet50_he_256_1'
  
  # Load the CIFAR10 data.
  (x_train, y_train), (x_test, y_test) = cifar10.load_data()
  
  # Normalize data.
  x_train = x_train.astype('float32') / 255
  x_test = x_test.astype('float32') / 255
  
  # If subtract pixel mean is enabled
  if subtract_pixel_mean:
      x_train_mean = np.mean(x_train, axis=0)
      x_train -= x_train_mean
      x_test -= x_train_mean
  
  print('x_train shape:', x_train.shape)
  print(x_train.shape[0], 'train samples')
  print(x_test.shape[0], 'test samples')
  print('y_train shape:', y_train.shape)
  
  # Convert class vectors to binary class matrices.
  y_train = keras.utils.to_categorical(y_train, num_classes)
  y_test = keras.utils.to_categorical(y_test, num_classes)
  
  if args.resume:
      print('resume from checkpoint')
      message = job_name + ' b_end'
      send_signal.send(args.node, 10002, message)
      model = keras.models.load_model(save_file)
      message = job_name + ' c_end'
      send_signal.send(args.node, 10002, message)
  else:
      print('train from start')
      model = models.Sequential()
      
      if '50' in args_model:
          base_model = ResNet50(weights=None, include_top=False, input_shape=(32, 32, 3), pooling=None)
      elif '101' in args_model:
          base_model = ResNet101(weights=None, include_top=False, input_shape=(32, 32, 3), pooling=None)
      elif '152' in args_model:
          base_model = ResNet152(weights=None, include_top=False, input_shape=(32, 32, 3), pooling=None)
      
      #base_model.summary()
      
      #pdb.set_trace()
      
      #model.add(layers.UpSampling2D((2,2)))
      #model.add(layers.UpSampling2D((2,2)))
      #model.add(layers.UpSampling2D((2,2)))
      model.add(base_model)
      model.add(layers.Flatten())
      #model.add(layers.BatchNormalization())
      #model.add(layers.Dense(128, activation='relu'))
      #model.add(layers.Dropout(0.5))
      #model.add(layers.BatchNormalization())
      #model.add(layers.Dense(64, activation='relu'))
      #model.add(layers.Dropout(0.5))
      #model.add(layers.BatchNormalization())
      model.add(layers.Dense(10, activation='softmax'))#, kernel_initializer='he_uniform'))
      
      model.compile(loss='categorical_crossentropy',
                    optimizer=Adam(lr=args_lr),
                    metrics=['accuracy'])
      
      #model.summary()
      print(model_type)
  
  #pdb.set_trace()
  
  current_epoch = 0
  
  ################### connects interrupt signal to the process #####################
  
  def terminateProcess(signalNumber, frame):
      # first record the wasted epoch time
      global epoch_begin_time
      if epoch_begin_time == 0:
          epoch_waste_time = 0
      else:
          epoch_waste_time = int(time.time() - epoch_begin_time)
  
      message = job_name + ' waste ' + str(epoch_waste_time) # 'job50 waste 100'
      if epoch_waste_time > 0:
          send_signal.send(args.node, 10002, message)
  
      print('checkpointing the model triggered by kill -15 signal')
      # delete whatever checkpoint that already exists
      for f in glob.glob(save_files):
          os.remove(f)
      model.save('/scratch/li.baol/checkpoint_final4/' + job_name + '_' + str(current_epoch) + '.h5')
      print ('(SIGTERM) terminating the process')
  
      message = job_name + ' checkpoint'
      send_signal.send(args.node, 10002, message)
  
      sys.exit()
  
  signal.signal(signal.SIGTERM, terminateProcess)
  
  #################################################################################
  
  logdir = '/scratch/li.baol/tsrbrd_log/job_runs/' + model_type + '/' + job_name
  
  tensorboard_callback = TensorBoard(log_dir=logdir)#, update_freq='batch')
  
  first_epoch_start = 0
  
  class PrintEpoch(keras.callbacks.Callback):
      def on_epoch_begin(self, epoch, logs=None):
          global current_epoch, first_epoch_start
          #remaining_epochs = epochs - epoch
          current_epoch = epoch
          print('current epoch ' + str(current_epoch))
          global epoch_begin_time
          epoch_begin_time = time.time()
          if epoch == starting_epoch and args.resume:
              first_epoch_start = time.time()
              message = job_name + ' d_end'
              send_signal.send(args.node, 10002, message)
          elif epoch == starting_epoch:
              first_epoch_start = time.time()           
          if epoch == starting_epoch:
              # send signal to indicate checkpoint is qualified
              message = job_name + ' ckpt_qual'
              send_signal.send(args.node, 10002, message)
  
  
      def on_epoch_end(self, epoch, logs=None):
          if epoch == starting_epoch:
              first_epoch_time = int(time.time() - first_epoch_start)
              message = job_name + ' 1st_epoch ' + str(first_epoch_time)
              send_signal.send(args.node, 10002, message)
          progress = round((epoch+1) / round(total_epochs/2), 2)
          message = job_name + ' completion ' + str(progress)
          send_signal.send(args.node, 10002, message)
  
  my_callback = PrintEpoch()
  
  callbacks = [tensorboard_callback, my_callback]
   #[checkpoint, lr_reducer, lr_scheduler, tensorboard_callback]
  
  # Run training
  
  model.fit(x_train, y_train,
            batch_size=batch_size,
            epochs=round(total_epochs/2),
            validation_data=(x_test, y_test),
            shuffle=True,
            callbacks=callbacks,
            initial_epoch=starting_epoch,
            verbose=1
            )
  
  # Score trained model.
  scores = model.evaluate(x_test, y_test, verbose=1)
  print('Test loss:', scores[0])
  print('Test accuracy:', scores[1])
  
  # send signal to indicate job has finished
  message = job_name + ' finish'
  send_signal.send(args.node, 10002, message)
   import pandas
  import pdb
  from datetime import datetime
  import matplotlib
  import numpy as np
  import matplotlib
  matplotlib.use('Agg')
  import matplotlib.pyplot as plt
  import glob
  import sys
  from matplotlib.ticker import MultipleLocator
  
  testcase = sys.argv[1] # K80_vgg19_32
  print(testcase)
  base_dir = '/scratch/li.baol/GPU_pwr_meas/tensorflow/round1/'
  log_dir = base_dir + testcase + '_*/' # /scratch/li.baol/GPU_pwr_meas/pytorch/K80_vgg19_32_*/
  dirs = glob.glob(log_dir)
  dirs.sort()
  pwr_all = []
  avg_all = []
  
  for tc in dirs:
      model = tc.split('/')[5+1]
      files = glob.glob(tc + "sample*.csv")
      files.sort()
      avg_pwr = [0] * (len(files) + 1)
      
      for fil in files:
          file_path = fil
          minute = int(fil.split('/')[6+1].split('_')[1].split('.')[0])
          try: # in case the file is empty
              data = pandas.read_csv(file_path)
              pwr = data[data.columns[2]].tolist()
              
              pwr_array = np.asarray(pwr)
              if (len(pwr) == 0):
                  avg_pwr[minute] = 0
              else:
                  avg_pwr[minute] = np.average(pwr_array)
          except pandas.errors.EmptyDataError:
              avg_pwr[minute] = 0
              pass
      pwr_all.append(avg_pwr)
      avg_pwr_filter = [i for i in avg_pwr if i > 10] # remove power measurements below 10W
      avg_all.append(sum(avg_pwr_filter) / len(avg_pwr_filter))
  
  
  #------------- plot ---------------#
  
  width = 0.1
  
  fig, axs = plt.subplots(1, 1, gridspec_kw={'hspace': 0, 'wspace': 0}, figsize=(12,5))
  fig.suptitle(testcase +  " GPU power (W) during training epochs")
  for i in range(len(pwr_all)):    
      x = np.arange(len(pwr_all[i]))
      axs.scatter(x, pwr_all[i], label = str(i))
  
  axs.set_xlabel('# of sample with 10s interval')
  axs.set_ylabel('power(W)')
  #axs.set_yticks(minor=True)
  axs.get_yaxis().set_minor_locator(MultipleLocator(5))
  axs.legend()
  
  axs.grid(which='both', axis='y', linestyle=':', color='black')
  pwr = int(sum(avg_all) / len(avg_all))
  plt.savefig(base_dir + "png/" + testcase + '_pwr' + str(pwr) + ".png")
  
  df = pandas.DataFrame(avg_all, columns=["power(W)"])
  df.to_csv(base_dir + 'csv/' + testcase + '.csv', index=False)
   """
  #Trains a ResNet on the CIFAR10 dataset.
  
  """
  
  from __future__ import print_function
  import keras
  from keras.layers import Dense, Conv2D, BatchNormalization, Activation
  from keras.layers import AveragePooling2D, Input, Flatten
  from keras.optimizers import Adam
  from keras.callbacks import ModelCheckpoint, LearningRateScheduler
  from keras.callbacks import ReduceLROnPlateau, TensorBoard
  from keras.preprocessing.image import ImageDataGenerator
  from keras.regularizers import l2
  from keras import backend as K
  from keras.models import Model
  from keras.datasets import cifar10
  from keras.applications.resnet import ResNet50, ResNet101, ResNet152
  from keras import models, layers, optimizers
  from datetime import datetime
  import tensorflow as tf
  import numpy as np
  import os
  import pdb
  import sys
  import argparse
  import time
  import signal
  import glob
  import json
  import send_signal
  
  parser = argparse.ArgumentParser(description='Tensorflow Cifar10 Training')
  parser.add_argument('--tc', metavar='TESTCASE', type=str, help='specific testcase name')
  parser.add_argument('--resume', dest='resume', action='store_true', help='if True, resume training from a checkpoint')
  parser.add_argument('--gpu_num', metavar='GPU_NUMBER', type=str, help='select which gpu to use')
  parser.add_argument('--node', metavar='HOST_NODE', type=str, help='node of the host (scheduler)')
  parser.set_defaults(resume=False)
  args = parser.parse_args()
  
  os.environ["CUDA_DEVICE_ORDER"]="PCI_BUS_ID"
  os.environ["CUDA_VISIBLE_DEVICES"]=args.gpu_num
  
  # Training parameters
  batch_size = 256
  args_lr = 0.01
  args_model = 'resnet50'
  
  epoch_begin_time = 0
  
  job_name = sys.argv[0].split('.')[0]
  save_files = '/scratch/li.baol/checkpoint_no_threshold/' + job_name + '*'
  
  total_epochs = 11
  starting_epoch = 0
  
  # first step is to update the PID
  pid = os.getpid()
  message = job_name + ' pid ' + str(pid) # 'job50 pid 3333'
  send_signal.send(args.node, 10002, message)
  
  if args.resume:
      save_file = glob.glob(save_files)[0]
  #    epochs = int(save_file.split('/')[4].split('_')[1].split('.')[0])
      starting_epoch = int(save_file.split('/')[4].split('.')[0].split('_')[-1])
  
  data_augmentation = True
  num_classes = 10
  
  # Subtracting pixel mean improves accuracy
  subtract_pixel_mean = True
  
  n = 3
  
  # Model name, depth and version
  model_type = args.tc #'P100_resnet50_he_256_1'
  
  # Load the CIFAR10 data.
  (x_train, y_train), (x_test, y_test) = cifar10.load_data()
  
  # Normalize data.
  x_train = x_train.astype('float32') / 255
  x_test = x_test.astype('float32') / 255
  
  # If subtract pixel mean is enabled
  if subtract_pixel_mean:
      x_train_mean = np.mean(x_train, axis=0)
      x_train -= x_train_mean
      x_test -= x_train_mean
  
  print('x_train shape:', x_train.shape)
  print(x_train.shape[0], 'train samples')
  print(x_test.shape[0], 'test samples')
  print('y_train shape:', y_train.shape)
  
  # Convert class vectors to binary class matrices.
  y_train = keras.utils.to_categorical(y_train, num_classes)
  y_test = keras.utils.to_categorical(y_test, num_classes)
  
  if args.resume:
      print('resume from checkpoint')
      message = job_name + ' b_end'
      send_signal.send(args.node, 10002, message)
      model = keras.models.load_model(save_file)
      message = job_name + ' c_end'
      send_signal.send(args.node, 10002, message)
  else:
      print('train from start')
      model = models.Sequential()
      
      if '50' in args_model:
          base_model = ResNet50(weights=None, include_top=False, input_shape=(32, 32, 3), pooling=None)
      elif '101' in args_model:
          base_model = ResNet101(weights=None, include_top=False, input_shape=(32, 32, 3), pooling=None)
      elif '152' in args_model:
          base_model = ResNet152(weights=None, include_top=False, input_shape=(32, 32, 3), pooling=None)
      
      #base_model.summary()
      
      #pdb.set_trace()
      
      #model.add(layers.UpSampling2D((2,2)))
      #model.add(layers.UpSampling2D((2,2)))
      #model.add(layers.UpSampling2D((2,2)))
      model.add(base_model)
      model.add(layers.Flatten())
      #model.add(layers.BatchNormalization())
      #model.add(layers.Dense(128, activation='relu'))
      #model.add(layers.Dropout(0.5))
      #model.add(layers.BatchNormalization())
      #model.add(layers.Dense(64, activation='relu'))
      #model.add(layers.Dropout(0.5))
      #model.add(layers.BatchNormalization())
      model.add(layers.Dense(10, activation='softmax'))#, kernel_initializer='he_uniform'))
      
      model.compile(loss='categorical_crossentropy',
                    optimizer=Adam(lr=args_lr),
                    metrics=['accuracy'])
      
      #model.summary()
      print(model_type)
  
  #pdb.set_trace()
  
  current_epoch = 0
  
  ################### connects interrupt signal to the process #####################
  
  def terminateProcess(signalNumber, frame):
      # first record the wasted epoch time
      global epoch_begin_time
      if epoch_begin_time == 0:
          epoch_waste_time = 0
      else:
          epoch_waste_time = int(time.time() - epoch_begin_time)
  
      message = job_name + ' waste ' + str(epoch_waste_time) # 'job50 waste 100'
      if epoch_waste_time > 0:
          send_signal.send(args.node, 10002, message)
  
      print('checkpointing the model triggered by kill -15 signal')
      # delete whatever checkpoint that already exists
      for f in glob.glob(save_files):
          os.remove(f)
      model.save('/scratch/li.baol/checkpoint_no_threshold/' + job_name + '_' + str(current_epoch) + '.h5')
      print ('(SIGTERM) terminating the process')
  
      message = job_name + ' checkpoint'
      send_signal.send(args.node, 10002, message)
  
      sys.exit()
  
  signal.signal(signal.SIGTERM, terminateProcess)
  
  #################################################################################
  
  logdir = '/scratch/li.baol/tsrbrd_log/job_runs/' + model_type + '/' + job_name
  
  tensorboard_callback = TensorBoard(log_dir=logdir)#, update_freq='batch')
  
  first_epoch_start = 0
  
  class PrintEpoch(keras.callbacks.Callback):
      def on_epoch_begin(self, epoch, logs=None):
          global current_epoch, first_epoch_start
          #remaining_epochs = epochs - epoch
          current_epoch = epoch
          print('current epoch ' + str(current_epoch))
          global epoch_begin_time
          epoch_begin_time = time.time()
          if epoch == starting_epoch and args.resume:
              first_epoch_start = time.time()
              message = job_name + ' d_end'
              send_signal.send(args.node, 10002, message)
          elif epoch == starting_epoch:
              first_epoch_start = time.time()           
          if epoch == starting_epoch:
              # send signal to indicate checkpoint is qualified
              message = job_name + ' ckpt_qual'
              send_signal.send(args.node, 10002, message)
  
  
      def on_epoch_end(self, epoch, logs=None):
          if epoch == starting_epoch:
              first_epoch_time = int(time.time() - first_epoch_start)
              message = job_name + ' 1st_epoch ' + str(first_epoch_time)
              send_signal.send(args.node, 10002, message)
          progress = round((epoch+1) / round(total_epochs/2), 2)
          message = job_name + ' completion ' + str(progress)
          send_signal.send(args.node, 10002, message)
  
  my_callback = PrintEpoch()
  
  callbacks = [tensorboard_callback, my_callback]
   #[checkpoint, lr_reducer, lr_scheduler, tensorboard_callback]
  
  # Run training
  
  model.fit(x_train, y_train,
            batch_size=batch_size,
            epochs=round(total_epochs/2),
            validation_data=(x_test, y_test),
            shuffle=True,
            callbacks=callbacks,
            initial_epoch=starting_epoch,
            verbose=1
            )
  
  # Score trained model.
  scores = model.evaluate(x_test, y_test, verbose=1)
  print('Test loss:', scores[0])
  print('Test accuracy:', scores[1])
  
  # send signal to indicate job has finished
  message = job_name + ' finish'
  send_signal.send(args.node, 10002, message)
  
apis: numpy.mean
apis: numpy.mean
apis: numpy.mean
apis: sklearn.metrics.mean_squared_error sklearn.neighbors.KNeighborsRegressor pandas.read_csv sklearn.linear_model.LinearRegression matplotlib.use sklearn.model_selection.train_test_split numpy.mean
apis: numpy.mean
apis: numpy.mean
apis: pandas.read_csv pandas.DataFrame numpy.asarray matplotlib.pyplot.subplots matplotlib.use matplotlib.ticker.MultipleLocator numpy.average
apis: numpy.mean