code: def train(rank, world_size, args):
      dist.init_process_group("nccl", rank=rank, world_size=world_size)
  
      if rank == 0 and args.logging_dir:
          if not os.path.isdir(args.logging_dir):
              os.makedirs(args.logging_dir)
          filehandler = logging.FileHandler(
              os.path.join(args.logging_dir, "train.log")
          )
          filehandler.setLevel(logging.INFO)
          logger.addHandler(filehandler)
  
          writer = SummaryWriter(log_dir=args.logging_dir)
      else:
          writer = None
  
      torch.manual_seed(0)
  
      torch.cuda.set_device(rank)
  
      symbols = get_symbol_list(args.text_preprocessor)
  
      model = Tacotron2(
          mask_padding=args.mask_padding,
          n_mels=args.n_mels,
          n_symbol=len(symbols),
          n_frames_per_step=args.n_frames_per_step,
          symbol_embedding_dim=args.symbols_embedding_dim,
          encoder_embedding_dim=args.encoder_embedding_dim,
          encoder_n_convolution=args.encoder_n_convolution,
          encoder_kernel_size=args.encoder_kernel_size,
          decoder_rnn_dim=args.decoder_rnn_dim,
          decoder_max_step=args.decoder_max_step,
          decoder_dropout=args.decoder_dropout,
          decoder_early_stopping=(not args.decoder_no_early_stopping),
          attention_rnn_dim=args.attention_rnn_dim,
          attention_hidden_dim=args.attention_hidden_dim,
          attention_location_n_filter=args.attention_location_n_filter,
          attention_location_kernel_size=args.attention_location_kernel_size,
          attention_dropout=args.attention_dropout,
          prenet_dim=args.prenet_dim,
          postnet_n_convolution=args.postnet_n_convolution,
          postnet_kernel_size=args.postnet_kernel_size,
          postnet_embedding_dim=args.postnet_embedding_dim,
          gate_threshold=args.gate_threshold,
      ).cuda(rank)
      model = torch.nn.SyncBatchNorm.convert_sync_batchnorm(model)
      model = torch.nn.parallel.DistributedDataParallel(
          model, device_ids=[rank]
      )
  
      optimizer = Adam(model.parameters(), lr=args.learning_rate)
  
      best_loss = float("inf")
      start_epoch = 0
  
      if args.checkpoint_path and os.path.isfile(args.checkpoint_path):
          logger.info(f"Checkpoint: loading '{args.checkpoint_path}'")
          map_location = {"cuda:%d" % 0: "cuda:%d" % rank}
          checkpoint = torch.load(
              args.checkpoint_path, map_location=map_location
          )
  
          start_epoch = checkpoint["epoch"]
          best_loss = checkpoint["best_loss"]
  
          model.load_state_dict(checkpoint["state_dict"])
          optimizer.load_state_dict(checkpoint["optimizer"])
  
          logger.info(
              f"Checkpoint: loaded '{args.checkpoint_path}' at epoch {checkpoint['epoch']}"
          )
  
      trainset, valset = get_datasets(args)
  
      train_sampler = torch.utils.data.distributed.DistributedSampler(
          trainset,
          shuffle=True,
          num_replicas=world_size,
          rank=rank,
      )
      val_sampler = torch.utils.data.distributed.DistributedSampler(
          valset,
          shuffle=False,
          num_replicas=world_size,
          rank=rank,
      )
  
      loader_params = {
          "batch_size": args.batch_size,
          "num_workers": args.workers,
          "prefetch_factor": 1024,
          "persistent_workers": True,
          "shuffle": False,
          "pin_memory": True,
          "drop_last": False,
          "collate_fn": partial(
              text_mel_collate_fn,
              n_frames_per_step=args.n_frames_per_step,
          ),
      }
  
      train_loader = DataLoader(
          trainset, sampler=train_sampler, **loader_params
      )
      val_loader = DataLoader(
          valset, sampler=val_sampler, **loader_params
      )
      dist.barrier()
  
      for epoch in range(start_epoch, args.epochs):
          start = time()
  
          model.train()
          trn_loss, counts = 0, 0
  
          if rank == 0:
              iterator = tqdm(
                  enumerate(train_loader),
                  desc=f"Epoch {epoch}",
                  total=len(train_loader),
              )
          else:
              iterator = enumerate(train_loader)
  
          for i, batch in iterator:
              adjust_learning_rate(
                  epoch,
                  optimizer,
                  args.learning_rate,
                  args.anneal_steps,
                  args.anneal_factor,
              )
  
              model.zero_grad()
  
              loss, losses = training_step(model, batch, i)
  
              loss.backward()
              torch.nn.utils.clip_grad_norm_(
                  model.parameters(), args.grad_clip
              )
  
              optimizer.step()
  
              if rank == 0 and writer:
                  global_iters = epoch * len(train_loader)
                  writer.add_scalar(
                      "trn/mel_loss", losses[0], global_iters
                  )
                  writer.add_scalar(
                      "trn/mel_postnet_loss", losses[1], global_iters
                  )
                  writer.add_scalar(
                      "trn/gate_loss", losses[2], global_iters
                  )
  
              trn_loss += loss * len(batch[0])
              counts += len(batch[0])
  
          trn_loss = trn_loss / counts
  
          trn_loss = reduce_tensor(trn_loss, world_size)
          if rank == 0:
              logger.info(
                  f"[Epoch: {epoch}] time: {time()-start}; trn_loss: {trn_loss}"
              )
              if writer:
                  writer.add_scalar("trn_loss", trn_loss, epoch)
  
          if ((epoch + 1) % args.validate_and_checkpoint_freq == 0) or (
              epoch == args.epochs - 1
          ):
              val_start_time = time()
              model.eval()
  
              val_loss, counts = 0, 0
              iterator = tqdm(
                  enumerate(val_loader),
                  desc=f"[Rank: {rank}; Epoch: {epoch}; Eval]",
                  total=len(val_loader),
              )
  
              with torch.no_grad():
                  for val_batch_idx, val_batch in iterator:
                      val_loss = val_loss + validation_step(
                          model, val_batch, val_batch_idx
                      )[0] * len(val_batch[0])
                      counts = counts + len(val_batch[0])
                  val_loss = val_loss / counts
  
              val_loss = reduce_tensor(val_loss, world_size)
              if rank == 0 and writer:
                  writer.add_scalar("val_loss", val_loss, epoch)
                  log_additional_info(writer, model, val_loader, epoch)
  
              if rank == 0:
                  is_best = val_loss < best_loss
                  best_loss = min(val_loss, best_loss)
                  logger.info(
                      f"[Rank: {rank}, Epoch: {epoch}; Eval] time: {time()-val_start_time}; val_loss: {val_loss}"
                  )
                  logger.info(
                      f"[Epoch: {epoch}] Saving checkpoint to {args.checkpoint_path}"
                  )
                  save_checkpoint(
                      {
                          "epoch": epoch + 1,
                          "state_dict": model.state_dict(),
                          "best_loss": best_loss,
                          "optimizer": optimizer.state_dict(),
                      },
                      is_best,
                      args.checkpoint_path,
                  )
  
      dist.destroy_process_group()
  
imports: import argparse
  import logging
  import os
  import random
  from datetime import datetime
  from functools import partial
  from time import time
  import matplotlib.pyplot
  import torch
  import torch.distributed
  import torch.multiprocessing
  import torchaudio
  from torch.optim import Adam
  from torch.utils.data import DataLoader
  from torch.utils.tensorboard import SummaryWriter
  from torchaudio.models import Tacotron2
  from tqdm import tqdm
  from datasets import SpectralNormalization, split_process_dataset, text_mel_collate_fn
  from loss import Tacotron2Loss
  from text.text_preprocessing import available_phonemizers, available_symbol_set, get_symbol_list, text_to_sequence
  from utils import save_checkpoint
label_desc: Use image node_id 10 for calling a global function with example usage: train(rank, world_size, args) without return types