code: def main():
      opt = parse_args()
  
      makedirs(opt.save_dir)
  
      filehandler = logging.FileHandler(
          os.path.join(opt.save_dir, opt.logging_file)
      )
      streamhandler = logging.StreamHandler()
      logger = logging.getLogger("")
      logger.setLevel(logging.INFO)
      logger.addHandler(filehandler)
      logger.addHandler(streamhandler)
      logger.info(opt)
  
      sw = SummaryWriter(
          logdir=opt.save_dir, flush_secs=5, verbose=False
      )
  
      if opt.kvstore is not None:
          kv = mx.kvstore.create(opt.kvstore)
          logger.info(
              "Distributed training with %d workers and current rank is %d"
              % (kv.num_workers, kv.rank)
          )
      if opt.use_amp:
          amp.init()
  
      batch_size = opt.batch_size
      classes = opt.num_classes
  
      num_gpus = opt.num_gpus
      batch_size *= max(1, num_gpus)
      logger.info(
          "Total batch size is set to %d on %d GPUs"
          % (batch_size, num_gpus)
      )
      context = (
          [mx.gpu(i) for i in range(num_gpus)]
          if num_gpus > 0
          else [mx.cpu()]
      )
      num_workers = opt.num_workers
  
      lr_decay = opt.lr_decay
      lr_decay_period = opt.lr_decay_period
      if opt.lr_decay_period > 0:
          lr_decay_epoch = list(
              range(lr_decay_period, opt.num_epochs, lr_decay_period)
          )
      else:
          lr_decay_epoch = [
              int(i) for i in opt.lr_decay_epoch.split(",")
          ]
      lr_decay_epoch = [e - opt.warmup_epochs for e in lr_decay_epoch]
  
      if opt.slowfast:
          optimizer = "nag"
      else:
          optimizer = "sgd"
  
      if opt.clip_grad > 0:
          optimizer_params = {
              "learning_rate": opt.lr,
              "wd": opt.wd,
              "momentum": opt.momentum,
              "clip_gradient": opt.clip_grad,
          }
      else:
          optimizer_params = {
              "learning_rate": opt.lr,
              "wd": opt.wd,
              "momentum": opt.momentum,
          }
  
      if opt.dtype != "float32":
          optimizer_params["multi_precision"] = True
  
      model_name = opt.model
      if opt.use_pretrained and len(opt.hashtag) > 0:
          opt.use_pretrained = opt.hashtag
      net = get_model(
          name=model_name,
          nclass=classes,
          pretrained=opt.use_pretrained,
          use_tsn=opt.use_tsn,
          num_segments=opt.num_segments,
          partial_bn=opt.partial_bn,
          bn_frozen=opt.freeze_bn,
      )
      net.cast(opt.dtype)
      net.collect_params().reset_ctx(context)
      logger.info(net)
  
      if opt.resume_params != "":
          net.load_parameters(opt.resume_params, ctx=context)
          print(
              "Continue training from model %s." % (opt.resume_params)
          )
  
      if opt.kvstore is not None:
          train_data, val_data, batch_fn = get_data_loader(
              opt, batch_size, num_workers, logger, kv
          )
      else:
          train_data, val_data, batch_fn = get_data_loader(
              opt, batch_size, num_workers, logger
          )
  
      num_batches = len(train_data) // opt.accumulate
      lr_scheduler = LRSequential(
          [
              LRScheduler(
                  "linear",
                  base_lr=opt.warmup_lr,
                  target_lr=opt.lr,
                  nepochs=opt.warmup_epochs,
                  iters_per_epoch=num_batches,
              ),
              LRScheduler(
                  opt.lr_mode,
                  base_lr=opt.lr,
                  target_lr=0,
                  nepochs=opt.num_epochs - opt.warmup_epochs,
                  iters_per_epoch=num_batches,
                  step_epoch=lr_decay_epoch,
                  step_factor=lr_decay,
                  power=2,
              ),
          ]
      )
      optimizer_params["lr_scheduler"] = lr_scheduler
  
      train_metric = mx.metric.Accuracy()
      acc_top1 = mx.metric.Accuracy()
      acc_top5 = mx.metric.TopKAccuracy(5)
  
      def test(ctx, val_data, kvstore=None):
          acc_top1.reset()
          acc_top5.reset()
          L = gluon.loss.SoftmaxCrossEntropyLoss()
          num_test_iter = len(val_data)
          val_loss_epoch = 0
          for i, batch in enumerate(val_data):
              data, label = batch_fn(batch, ctx)
              outputs = []
              for _, X in enumerate(data):
                  X = X.reshape((-1,) + X.shape[2:])
                  pred = net(X.astype(opt.dtype, copy=False))
                  outputs.append(pred)
  
              loss = [
                  L(yhat, y.astype(opt.dtype, copy=False))
                  for yhat, y in zip(outputs, label)
              ]
  
              acc_top1.update(label, outputs)
              acc_top5.update(label, outputs)
  
              val_loss_epoch += sum(
                  [l.mean().asscalar() for l in loss]
              ) / len(loss)
  
              if opt.log_interval and not (i + 1) % opt.log_interval:
                  _, top1 = acc_top1.get()
                  _, top5 = acc_top5.get()
                  logger.info(
                      "Batch [%04d]/[%04d]: acc-top1=%f acc-top5=%f"
                      % (i, num_test_iter, top1 * 100, top5 * 100)
                  )
  
          _, top1 = acc_top1.get()
          _, top5 = acc_top5.get()
          val_loss = val_loss_epoch / num_test_iter
  
          if kvstore is not None:
              top1_nd = nd.zeros(1)
              top5_nd = nd.zeros(1)
              val_loss_nd = nd.zeros(1)
              kvstore.push(111111, nd.array(np.array([top1])))
              kvstore.pull(111111, out=top1_nd)
              kvstore.push(555555, nd.array(np.array([top5])))
              kvstore.pull(555555, out=top5_nd)
              kvstore.push(999999, nd.array(np.array([val_loss])))
              kvstore.pull(999999, out=val_loss_nd)
              top1 = top1_nd.asnumpy() / kvstore.num_workers
              top5 = top5_nd.asnumpy() / kvstore.num_workers
              val_loss = val_loss_nd.asnumpy() / kvstore.num_workers
  
          return (top1, top5, val_loss)
  
      def train(ctx):
          if isinstance(ctx, mx.Context):
              ctx = [ctx]
  
          if opt.no_wd:
              for k, v in net.collect_params(
                  ".*beta|.*gamma|.*bias"
              ).items():
                  v.wd_mult = 0.0
  
          if opt.partial_bn:
              train_patterns = None
              if "inceptionv3" in opt.model:
                  train_patterns = ".*weight|.*bias|inception30_batchnorm0_gamma|inception30_batchnorm0_beta|inception30_batchnorm0_running_mean|inception30_batchnorm0_running_var"
              elif "inceptionv1" in opt.model:
                  train_patterns = ".*weight|.*bias|googlenet0_batchnorm0_gamma|googlenet0_batchnorm0_beta|googlenet0_batchnorm0_running_mean|googlenet0_batchnorm0_running_var"
              else:
                  logger.info(
                      "Current model does not support partial batch normalization."
                  )
  
              if opt.kvstore is not None:
                  trainer = gluon.Trainer(
                      net.collect_params(train_patterns),
                      optimizer,
                      optimizer_params,
                      kvstore=kv,
                      update_on_kvstore=False,
                  )
              else:
                  trainer = gluon.Trainer(
                      net.collect_params(train_patterns),
                      optimizer,
                      optimizer_params,
                      update_on_kvstore=False,
                  )
          elif opt.freeze_bn:
              train_patterns = ".*weight|.*bias"
              if opt.kvstore is not None:
                  trainer = gluon.Trainer(
                      net.collect_params(train_patterns),
                      optimizer,
                      optimizer_params,
                      kvstore=kv,
                      update_on_kvstore=False,
                  )
              else:
                  trainer = gluon.Trainer(
                      net.collect_params(train_patterns),
                      optimizer,
                      optimizer_params,
                      update_on_kvstore=False,
                  )
          else:
              if opt.kvstore is not None:
                  trainer = gluon.Trainer(
                      net.collect_params(),
                      optimizer,
                      optimizer_params,
                      kvstore=kv,
                      update_on_kvstore=False,
                  )
              else:
                  trainer = gluon.Trainer(
                      net.collect_params(),
                      optimizer,
                      optimizer_params,
                      update_on_kvstore=False,
                  )
  
          if opt.accumulate > 1:
              params = [
                  p
                  for p in net.collect_params().values()
                  if p.grad_req != "null"
              ]
              for p in params:
                  p.grad_req = "add"
  
          if opt.resume_states != "":
              trainer.load_states(opt.resume_states)
  
          if opt.use_amp:
              amp.init_trainer(trainer)
  
          L = gluon.loss.SoftmaxCrossEntropyLoss()
  
          best_val_score = 0
          lr_decay_count = 0
  
          for epoch in range(opt.resume_epoch, opt.num_epochs):
              tic = time.time()
              train_metric.reset()
              btic = time.time()
              num_train_iter = len(train_data)
              train_loss_epoch = 0
              train_loss_iter = 0
  
              for i, batch in enumerate(train_data):
                  data, label = batch_fn(batch, ctx)
  
                  with ag.record():
                      outputs = []
                      for _, X in enumerate(data):
                          X = X.reshape((-1,) + X.shape[2:])
                          pred = net(X.astype(opt.dtype, copy=False))
                          outputs.append(pred)
                      loss = [
                          L(yhat, y.astype(opt.dtype, copy=False))
                          for yhat, y in zip(outputs, label)
                      ]
  
                      if opt.use_amp:
                          with amp.scale_loss(
                              loss, trainer
                          ) as scaled_loss:
                              ag.backward(scaled_loss)
                      else:
                          ag.backward(loss)
  
                  if opt.accumulate > 1:
                      if (i + 1) % opt.accumulate == 0:
                          if opt.kvstore is not None:
                              trainer.step(
                                  batch_size
                                  * kv.num_workers
                                  * opt.accumulate
                              )
                          else:
                              trainer.step(batch_size * opt.accumulate)
                              net.collect_params().zero_grad()
                  else:
                      if opt.kvstore is not None:
                          trainer.step(batch_size * kv.num_workers)
                      else:
                          trainer.step(batch_size)
  
                  train_metric.update(label, outputs)
                  train_loss_iter = sum(
                      [l.mean().asscalar() for l in loss]
                  ) / len(loss)
                  train_loss_epoch += train_loss_iter
  
                  (
                      train_metric_name,
                      train_metric_score,
                  ) = train_metric.get()
                  sw.add_scalar(
                      tag="train_acc_top1_iter",
                      value=train_metric_score * 100,
                      global_step=epoch * num_train_iter + i,
                  )
                  sw.add_scalar(
                      tag="train_loss_iter",
                      value=train_loss_iter,
                      global_step=epoch * num_train_iter + i,
                  )
                  sw.add_scalar(
                      tag="learning_rate_iter",
                      value=trainer.learning_rate,
                      global_step=epoch * num_train_iter + i,
                  )
  
                  if (
                      opt.log_interval
                      and not (i + 1) % opt.log_interval
                  ):
                      logger.info(
                          "Epoch[%03d] Batch [%04d]/[%04d]\tSpeed: %f samples/sec\t %s=%f\t loss=%f\t lr=%f"
                          % (
                              epoch,
                              i,
                              num_train_iter,
                              batch_size
                              * opt.log_interval
                              / (time.time() - btic),
                              train_metric_name,
                              train_metric_score * 100,
                              train_loss_epoch / (i + 1),
                              trainer.learning_rate,
                          )
                      )
                      btic = time.time()
  
              train_metric_name, train_metric_score = train_metric.get()
              throughput = int(batch_size * i / (time.time() - tic))
              mx.ndarray.waitall()
  
              logger.info(
                  "[Epoch %03d] training: %s=%f\t loss=%f"
                  % (
                      epoch,
                      train_metric_name,
                      train_metric_score * 100,
                      train_loss_epoch / num_train_iter,
                  )
              )
              logger.info(
                  "[Epoch %03d] speed: %d samples/sec\ttime cost: %f"
                  % (epoch, throughput, time.time() - tic)
              )
              sw.add_scalar(
                  tag="train_loss_epoch",
                  value=train_loss_epoch / num_train_iter,
                  global_step=epoch,
              )
  
              if not opt.train_only:
                  if (
                      opt.kvstore is not None
                      and epoch == opt.resume_epoch
                  ):
                      kv.init(111111, nd.zeros(1))
                      kv.init(555555, nd.zeros(1))
                      kv.init(999999, nd.zeros(1))
  
                  if opt.kvstore is not None:
                      acc_top1_val, acc_top5_val, loss_val = test(
                          ctx, val_data, kv
                      )
                  else:
                      acc_top1_val, acc_top5_val, loss_val = test(
                          ctx, val_data
                      )
  
                  logger.info(
                      "[Epoch %03d] validation: acc-top1=%f acc-top5=%f loss=%f"
                      % (
                          epoch,
                          acc_top1_val * 100,
                          acc_top5_val * 100,
                          loss_val,
                      )
                  )
                  sw.add_scalar(
                      tag="val_loss_epoch",
                      value=loss_val,
                      global_step=epoch,
                  )
                  sw.add_scalar(
                      tag="val_acc_top1_epoch",
                      value=acc_top1_val * 100,
                      global_step=epoch,
                  )
  
                  if acc_top1_val > best_val_score:
                      best_val_score = acc_top1_val
                      net.save_parameters(
                          "%s/%.4f-%s-%s-%03d-best.params"
                          % (
                              opt.save_dir,
                              best_val_score,
                              opt.dataset,
                              model_name,
                              epoch,
                          )
                      )
                      trainer.save_states(
                          "%s/%.4f-%s-%s-%03d-best.states"
                          % (
                              opt.save_dir,
                              best_val_score,
                              opt.dataset,
                              model_name,
                              epoch,
                          )
                      )
                  else:
                      if (
                          opt.save_frequency
                          and opt.save_dir
                          and (epoch + 1) % opt.save_frequency == 0
                      ):
                          net.save_parameters(
                              "%s/%s-%s-%03d.params"
                              % (
                                  opt.save_dir,
                                  opt.dataset,
                                  model_name,
                                  epoch,
                              )
                          )
                          trainer.save_states(
                              "%s/%s-%s-%03d.states"
                              % (
                                  opt.save_dir,
                                  opt.dataset,
                                  model_name,
                                  epoch,
                              )
                          )
  
          # save the last model
          net.save_parameters(
              "%s/%s-%s-%03d.params"
              % (
                  opt.save_dir,
                  opt.dataset,
                  model_name,
                  opt.num_epochs - 1,
              )
          )
          trainer.save_states(
              "%s/%s-%s-%03d.states"
              % (
                  opt.save_dir,
                  opt.dataset,
                  model_name,
                  opt.num_epochs - 1,
              )
          )
  
      if opt.mode == "hybrid":
          net.hybridize(static_alloc=True, static_shape=True)
  
      train(context)
      sw.close()
  
file_path: gluon-cv/scripts/action-recognition/train_recognizer.py
imports: import argparse, time, logging, os, sys, math
  import numpy
  import mxnet
  import gluoncv
  from mxnet import gluon, nd, init, context
  from mxnet import autograd
  from mxnet.gluon import nn
  from mxnet.gluon.data.vision import transforms
  from mxboard import SummaryWriter
  from mxnet.contrib import amp
  from gluoncv.data.transforms import video
  from gluoncv.data import UCF101, Kinetics400, SomethingSomethingV2, HMDB51, VideoClsCustom
  from gluoncv.model_zoo import get_model
  from gluoncv.utils import makedirs, LRSequential, LRScheduler, split_and_load
  from gluoncv.data.sampler import SplitSampler, ShuffleSplitSampler
label_desc: Use image node_id 3 for calling a global function with example usage: main() and returns: top1,  top5,  val_loss