text: import sys
  import os
  import time
  import argparse
  import param
  import logging
  import pickle
  import numpy as np
  from threading import Thread
  
  logging.basicConfig(format='%(message)s', level=logging.INFO)
  
  def Run(args):
      # create a Clairvoyante
      logging.info("Initializing model ...")
      if args.v2 == True:
          import utils_v2 as utils
          if args.slim == True:
              import clairvoyante_v2_slim as cv
          else:
              import clairvoyante_v2 as cv
      elif args.v3 == True:
          import utils_v2 as utils # v3 network is using v2 utils
          if args.slim == True:
              import clairvoyante_v3_slim as cv
          else:
              import clairvoyante_v3 as cv
      utils.SetupEnv()
      m = cv.Clairvoyante()
      m.init()
  
      if args.ochk_prefix == None:
          sys.exit("--chk_prefix must be defined in nonstop training mode")
      if args.chkpnt_fn != None:
          m.restoreParameters(os.path.abspath(args.chkpnt_fn))
      TrainAll(args, m, utils)
  
  
  def TrainAll(args, m, utils):
      logging.info("Loading the training dataset ...")
      if args.bin_fn != None:
          with open(args.bin_fn, "rb") as fh:
              total = pickle.load(fh)
              XArrayCompressed = pickle.load(fh)
              YArrayCompressed = pickle.load(fh)
              posArrayCompressed = pickle.load(fh)
      else:
          total, XArrayCompressed, YArrayCompressed, posArrayCompressed = \
          utils.GetTrainingArray(args.tensor_fn,
                                 args.var_fn,
                                 args.bed_fn)
  
      logging.info("The size of training dataset: {}".format(total))
  
      # Op to write logs to Tensorboard
      if args.olog_dir != None:
          summaryWriter = m.summaryFileWriter(args.olog_dir)
  
      # Train and save the parameters, we train on the first 90% variant sites and validate on the last 10% variant sites
      logging.info("Start training ...")
      logging.info("Learning rate: %.2e" % m.setLearningRate(args.learning_rate))
      logging.info("L2 regularization lambda: %.2e" % m.setL2RegularizationLambda(args.lambd))
  
      # Model Constants
      trainingStart = time.time()
      trainingTotal = total
      maxLearningRateSwitch = param.maxLearningRateSwitch
      batchSize = param.trainBatchSize
  
      # Variables reset per epoch
      batchSize = param.trainBatchSize
      epochStart = time.time()
      trainLossSum = 0
      datasetPtr = 0
  
      i = 1 if args.chkpnt_fn == None else int(args.chkpnt_fn[-param.parameterOutputPlaceHolder:])+1
      XBatch, XNum, XEndFlag = utils.DecompressArray(XArrayCompressed, datasetPtr, batchSize, total)
      YBatch, YNum, YEndFlag = utils.DecompressArray(YArrayCompressed, datasetPtr, batchSize, total)
      datasetPtr += XNum
      while i < param.maxEpoch:
          threadPool = []
          threadPool.append(Thread(target=m.trainNoRT, args=(XBatch, YBatch, )))
  
          for t in threadPool: t.start()
  
          XBatch2, XNum2, XEndFlag2 = utils.DecompressArray(XArrayCompressed, datasetPtr, batchSize, total)
          YBatch2, YNum2, YEndFlag2 = utils.DecompressArray(YArrayCompressed, datasetPtr, batchSize, total)
          if XNum2 != YNum2 or XEndFlag2 != YEndFlag2:
              sys.exit("Inconsistency between decompressed arrays: %d/%d" % (XNum, YNum))
  
          for t in threadPool: t.join()
  
          XBatch = XBatch2; YBatch = YBatch2
          trainLossSum += m.trainLossRTVal
          summary = m.trainSummaryRTVal
          if args.olog_dir != None:
              summaryWriter.add_summary(summary, i)
          datasetPtr += XNum2
  
          if XEndFlag2 != 0:
              logging.info(" ".join([str(i), "Training loss:", str(trainLossSum/trainingTotal)]))
              logging.info("Epoch time elapsed: %.2f s" % (time.time() - epochStart))
              parameterOutputPath = "%s-%%0%dd" % ( args.ochk_prefix, param.parameterOutputPlaceHolder )
              m.saveParameters(parameterOutputPath % i)
  
              # Reset per epoch variables
              i += 1;
              trainLossSum = 0; datasetPtr = 0; epochStart = time.time(); batchSize = param.trainBatchSize
              XBatch, XNum, XEndFlag = utils.DecompressArray(XArrayCompressed, datasetPtr, batchSize, total)
              YBatch, YNum, YEndFlag = utils.DecompressArray(YArrayCompressed, datasetPtr, batchSize, total)
              datasetPtr += XNum
  
      logging.info("Training time elapsed: %.2f s" % (time.time() - trainingStart))
  
  
  def main():
      parser = argparse.ArgumentParser(
              description="Train Clairvoyante Nonstop" )
  
      parser.add_argument('--bin_fn', type=str, default = None,
              help="Binary tensor input generated by tensor2Bin.py, tensor_fn, var_fn and bed_fn will be ignored")
  
      parser.add_argument('--tensor_fn', type=str, default = "vartensors",
              help="Tensor input")
  
      parser.add_argument('--var_fn', type=str, default = "truthvars",
              help="Truth variants list input")
  
      parser.add_argument('--bed_fn', type=str, default = None,
              help="High confident genome regions input in the BED format")
  
      parser.add_argument('--chkpnt_fn', type=str, default = None,
              help="Input a checkpoint for testing or continue training")
  
      parser.add_argument('--learning_rate', type=float, default = param.initialLearningRate,
              help="Set the initial learning rate, default: %(default)s")
  
      parser.add_argument('--lambd', type=float, default = param.l2RegularizationLambda,
              help="Set the l2 regularization lambda, default: %(default)s")
  
      parser.add_argument('--ochk_prefix', type=str, default = None,
              help="Prefix for checkpoint outputs at each learning rate change, optional")
  
      parser.add_argument('--olog_dir', type=str, default = None,
              help="Directory for tensorboard log outputs, optional")
  
      parser.add_argument('--v3', type=param.str2bool, nargs='?', const=True, default = True,
              help="Use Clairvoyante version 3")
  
      parser.add_argument('--v2', type=param.str2bool, nargs='?', const=True, default = False,
              help="Use Clairvoyante version 2")
  
      parser.add_argument('--slim', type=param.str2bool, nargs='?', const=True, default = False,
              help="Train using the slim version of Clairvoyante, optional")
  
      args = parser.parse_args()
  
      if len(sys.argv[1:]) == 0:
          parser.print_help()
          sys.exit(1)
  
      Run(args)
  
  
  if __name__ == "__main__":
      main()
  