class_docstr: Tensorizer for token level classification tasks such as NER, POS etc
  using RoBERTa. Here each token has an associated label and the tensorizer
  should output a label tensor as well. The input for this tensorizer comes
  from the CoNLLUNERDataSource data source.
docstr: Numberize both the tokens and labels. Since we break up tokens, the label
  for anything other than the first sub-word is assigned the padding idx.
docstr_tok: ["Numberize","both","the","tokens","and","labels",".","Since","we","break","up","tokens",",","the","label","for","anything","other","than","the","first","sub-word","is","assigned","the","padding","idx","."]
code: def numberize(self, row: Dict) -> Tuple[Any, ...]:
      """
      Numberize both the tokens and labels. Since we break up tokens, the label
      for anything other than the first sub-word is assigned the padding idx.
      """
      # The relevant data source outputs a list of strings for both
      # the tokens and labels
      raw_tokens = row[self.columns[0]]
      raw_labels = row[self.labels_columns[0]]
  
      numberized_tokens = []
      numberized_labels = []
  
      for token, label in zip(raw_tokens, raw_labels):
          sub_tokens = self.tokenizer.tokenize(token)
  
          # convert token level labels to sub-word level labels
          for i in range(len(sub_tokens)):
              sub_token_id = self.vocab.lookup_all(sub_tokens[i].value)
              label_id = (
                  self.labels_vocab.lookup_all(label)
                  if i == 0
                  else self.labels_pad_idx
              )
  
              numberized_tokens.append(sub_token_id)
              numberized_labels.append(label_id)
  
      # Ensure the numberized labels and tokens are wrapped correctly
      numberized_sentences = [
          [self.vocab.get_bos_index()]
          + numberized_tokens[: self.max_seq_len - 2]
          + [self.vocab.get_eos_index()]
      ]
      numberized_labels = (
          [self.labels_pad_idx]
          + numberized_labels[: self.max_seq_len - 2]
          + [self.labels_pad_idx]
      )
  
      seq_lens = (len(sentence) for sentence in numberized_sentences)
      segment_labels = (
          [i] * seq_len for i, seq_len in enumerate(seq_lens)
      )
      tokens = list(itertools.chain(*numberized_sentences))
      segment_labels = list(itertools.chain(*segment_labels))
      seq_len = len(tokens)
      positions = [index for index in range(seq_len)]
      return (
          tokens,
          segment_labels,
          seq_len,
          positions,
          numberized_labels,
      )
  
imports: import itertools
  from typing import Any, Dict, List, Tuple
  import torch
  from pytext import resources
  from pytext.common.constants import SpecialTokens
  from pytext.config.component import ComponentType, create_component
  from pytext.data.bert_tensorizer import BERTTensorizerBase, BERTTensorizerBaseScriptImpl, build_fairseq_vocab
  from pytext.data.tokenizers import GPT2BPETokenizer, Tokenizer
  from pytext.data.utils import pad_and_tensorize, VocabBuilder, Vocabulary
  from pytext.torchscript.tensorizer import ScriptRoBERTaTensorizer
  from pytext.torchscript.vocab import ScriptVocabulary
  from pytext.utils.file_io import PathManager
label_desc: Use image node_id 5 for calling the RoBERTaTokenLevelTensorizer obj's underlying member method code with example usage: obj.numberize(row) and returns: tokens,  segment_labels,  seq_len,  positions,  numberized_labels