code: def token_from_subtoken(
      units: torch.Tensor, mask: torch.Tensor
  ) -> torch.Tensor:
      """Assemble token level units from subtoken level units
  
      Args:
          units: torch.Tensor of shape [batch_size, SUBTOKEN_seq_length, n_features]
          mask: mask of token beginnings. For example: for tokens
  
                  [[``[CLS]`` ``My``, ``capybara``, ``[SEP]``],
                  [``[CLS]`` ``Your``, ``aar``, ``##dvark``, ``is``, ``awesome``, ``[SEP]``]]
  
              the mask will be
  
                  [[0, 1, 1, 0, 0, 0, 0],
                  [0, 1, 1, 0, 1, 1, 0]]
  
      Returns:
          word_level_units: Units assembled from ones in the mask. For the
              example above this units will correspond to the following
  
                  [[``My``, ``capybara``],
                  [``Your`, ``aar``, ``is``, ``awesome``,]]
  
              the shape of this tensor will be [batch_size, TOKEN_seq_length, n_features]
      """
      shape = units.size()
      batch_size = shape[0]
      nf = shape[2]
      nf_int = units.size()[-1]
  
      token_seq_lengths = torch.sum(mask, 1).to(torch.int64)
  
      n_words = torch.sum(token_seq_lengths)
  
      max_token_seq_len = torch.max(token_seq_lengths)
  
      idxs = torch.stack(torch.nonzero(mask, as_tuple=True), dim=1)
  
      sample_ids_in_batch = torch.nn.functional.pad(
          input=idxs[:, 0], pad=[1, 0]
      )
  
      a = torch.logical_not(
          torch.eq(
              sample_ids_in_batch[1:], sample_ids_in_batch[:-1]
          ).to(torch.int64)
      )
  
      q = a * torch.arange(n_words).to(torch.int64)
      count_to_substract = torch.nn.functional.pad(
          torch.masked_select(q, q.to(torch.bool)), [1, 0]
      )
  
      new_word_indices = torch.arange(n_words).to(
          torch.int64
      ) - torch.gather(
          count_to_substract, dim=0, index=torch.cumsum(a, 0)
      )
  
      n_total_word_elements = (batch_size * max_token_seq_len).to(
          torch.int32
      )
      word_indices_flat = (
          idxs[:, 0] * max_token_seq_len + new_word_indices
      ).to(torch.int64)
      x_mask = torch.sum(
          torch.nn.functional.one_hot(
              word_indices_flat, n_total_word_elements
          ),
          0,
      )
      x_mask = x_mask.to(torch.bool)
  
      full_range = torch.arange(batch_size * max_token_seq_len).to(
          torch.int64
      )
      nonword_indices_flat = torch.masked_select(
          full_range, torch.logical_not(x_mask)
      )
  
      def gather_nd(params, indices):
          assert type(indices) == torch.Tensor
          return params[indices.transpose(0, 1).long().numpy().tolist()]
  
      elements = gather_nd(units, idxs)
  
      sh = tuple(
          torch.stack(
              [
                  torch.sum(max_token_seq_len - token_seq_lengths),
                  torch.tensor(nf),
              ],
              0,
          ).numpy()
      )
      paddings = torch.zeros(sh, dtype=torch.float64)
  
      def dynamic_stitch(indices, data):
          # https://discuss.pytorch.org/t/equivalent-of-tf-dynamic-partition/53735/2
          n = sum(idx.numel() for idx in indices)
          res = [None] * n
          for i, data_ in enumerate(data):
              idx = indices[i].view(-1)
              if idx.numel() > 0:
                  d = data_.view(idx.numel(), -1)
                  k = 0
                  for idx_ in idx:
                      res[idx_] = d[k].to(torch.float64)
                      k += 1
          return res
  
      tensor_flat = torch.stack(
          dynamic_stitch(
              [word_indices_flat, nonword_indices_flat],
              [elements, paddings],
          )
      )
  
      tensor = torch.reshape(
          tensor_flat, (batch_size, max_token_seq_len.item(), nf_int)
      )
  
      return tensor
  
file_path: DeepPavlov/deeppavlov/models/torch_bert/torch_transformers_sequence_tagger.py
imports: from logging import getLogger
  from pathlib import Path
  from typing import List, Union, Dict, Optional, Tuple
  import numpy
  import torch
  from transformers import AutoModelForTokenClassification, AutoConfig
  from deeppavlov.core.commands.utils import expand_path
  from deeppavlov.core.common.errors import ConfigError
  from deeppavlov.core.common.registry import register
  from deeppavlov.core.models.torch_model import TorchModel
  from deeppavlov.models.torch_bert.crf import CRF
label_desc: Use image node_id 1 for calling a global function with example usage: token_from_subtoken(units, mask) and returns: tensor, params, res