code: def tensorify(vocabs, minibatch, device, left_pad=False):
      """
      This function transforms a batch of example in tensors
      Each example looks like
      {'src': {'src': ..., 'feats': [...], 'src_ids': ...},
       'tgt': {'tgt': ..., 'tgt_ids': ...},
       'src_original': ['tok1', ...'tokn'],
       'tgt_original': ['tok1', ...'tokm'],
       'cid': corpus id
       'cid_line_number' : corpus id line number
       'ind_in_bucket': index in bucket
       'align': ...,
      }
      Returns  Dict of batch Tensors
          {'src': [seqlen, batchsize, n_feats+1],
           'tgt' : [seqlen, batchsize, n_feats=1],
           'cid': [batchsize],
           'cid_line_number' : [batchsize],
           'ind_in_bucket': [batchsize],
           'srclen': [batchsize],
           'tgtlen': [batchsize],
           'align': alignment sparse tensor
          }
      """
      tensor_batch = {}
      if left_pad:
          tbatchsrc = [
              torch.tensor(
                  ex["src"]["src_ids"], dtype=torch.long, device=device
              ).flip(dims=[0])
              for ex, indice in minibatch
          ]
      else:
          tbatchsrc = [
              torch.tensor(
                  ex["src"]["src_ids"], dtype=torch.long, device=device
              )
              for ex, indice in minibatch
          ]
      padidx = vocabs["src"][DefaultTokens.PAD]
      tbatchsrc = pad_sequence(
          tbatchsrc, batch_first=True, padding_value=padidx
      )
      if "feats" in minibatch[0][0]["src"]:
          tbatchfs = [tbatchsrc]
          for feat_id in range(len(minibatch[0][0]["src"]["feats"])):
              if left_pad:
                  tbatchfeat = [
                      torch.tensor(
                          ex["src"]["feats"][feat_id],
                          dtype=torch.long,
                          device=device,
                      ).flip(dims=[0])
                      for ex, indice in minibatch
                  ]
              else:
                  tbatchfeat = [
                      torch.tensor(
                          ex["src"]["feats"][feat_id],
                          dtype=torch.long,
                          device=device,
                      )
                      for ex, indice in minibatch
                  ]
              padidx = vocabs["src_feats"][feat_id][DefaultTokens.PAD]
              tbatchfeat = pad_sequence(
                  tbatchfeat, batch_first=True, padding_value=padidx
              )
              tbatchfs.append(tbatchfeat)
          tbatchsrc = torch.stack(tbatchfs, dim=2)
      else:
          # Need to add features in last dimensions
          tbatchsrc = tbatchsrc[:, :, None]
  
      if left_pad:
          tensor_batch["src"] = tbatchsrc.flip(dims=[1])
      else:
          tensor_batch["src"] = tbatchsrc
  
      tensor_batch["srclen"] = torch.tensor(
          [len(ex["src"]["src_ids"]) for ex, indice in minibatch],
          dtype=torch.long,
          device=device,
      )
  
      if minibatch[0][0]["tgt"] is not None:
          if left_pad:
              tbatchtgt = [
                  torch.tensor(
                      ex["tgt"]["tgt_ids"],
                      dtype=torch.long,
                      device=device,
                  ).flip(dims=[0])
                  for ex, indice in minibatch
              ]
          else:
              tbatchtgt = [
                  torch.tensor(
                      ex["tgt"]["tgt_ids"],
                      dtype=torch.long,
                      device=device,
                  )
                  for ex, indice in minibatch
              ]
  
          padidx = vocabs["tgt"][DefaultTokens.PAD]
          tbatchtgt = pad_sequence(
              tbatchtgt, batch_first=True, padding_value=padidx
          )
          tbatchtgt = tbatchtgt[:, :, None]
          tbatchtgtlen = torch.tensor(
              [len(ex["tgt"]["tgt_ids"]) for ex, indice in minibatch],
              dtype=torch.long,
              device=device,
          )
          if left_pad:
              tensor_batch["tgt"] = tbatchtgt.flip(dims=[1])
          else:
              tensor_batch["tgt"] = tbatchtgt
          tensor_batch["tgtlen"] = tbatchtgtlen
  
      if (
          "align" in minibatch[0][0].keys()
          and minibatch[0][0]["align"] is not None
      ):
          sparse_idx = []
          for i, (ex, indice) in enumerate(minibatch):
              for src, tgt in parse_align_idx(ex["align"]):
                  sparse_idx.append([i, tgt + 1, src])
          tbatchalign = torch.tensor(
              sparse_idx, dtype=torch.long, device=device
          )
          tensor_batch["align"] = tbatchalign
  
      if "src_map" in minibatch[0][0].keys():
          src_vocab_size = (
              max([max(ex["src_map"]) for ex, indice in minibatch]) + 1
          )
          src_map = torch.zeros(
              len(tensor_batch["srclen"]),
              tbatchsrc.size(1),
              src_vocab_size,
              device=device,
          )
          for i, (ex, indice) in enumerate(minibatch):
              for j, t in enumerate(ex["src_map"]):
                  src_map[i, j, t] = 1
          tensor_batch["src_map"] = src_map
  
      if "alignment" in minibatch[0][0].keys():
          alignment = torch.zeros(
              len(tensor_batch["srclen"]),
              tbatchtgt.size(1),
              dtype=torch.long,
              device=device,
          )
          for i, (ex, indice) in enumerate(minibatch):
              alignment[i, : len(ex["alignment"])] = torch.tensor(
                  ex["alignment"], dtype=torch.long, device=device
              )
          tensor_batch["alignment"] = alignment
  
      if "src_ex_vocab" in minibatch[0][0].keys():
          tensor_batch["src_ex_vocab"] = [
              ex["src_ex_vocab"] for ex, indice in minibatch
          ]
  
      tensor_batch["ind_in_bucket"] = [
          indice for ex, indice in minibatch
      ]
  
      tensor_batch["cid"] = [ex["cid"] for ex, indice in minibatch]
      tensor_batch["cid_line_number"] = [
          ex["cid_line_number"] for ex, indice in minibatch
      ]
  
      return tensor_batch
  
file_path: OpenNMT-py/onmt/inputters/text_utils.py
imports: import torch
  import pyonmttok
  from onmt.constants import DefaultTokens, CorpusTask, ModelTask
  from torch.nn.utils.rnn import pad_sequence
  from onmt.utils.logging import logger
  from collections import Counter
label_desc: Use image node_id 8 for calling a global function with example usage: tensorify(vocabs, minibatch, device, left_pad) and returns: tensor_batch