code: def get_features(
      queries: List[str],
      tokenizer: TokenizerSpec,
      max_seq_length: int = -1,
      label_ids: dict = None,
      pad_label: str = "O",
      raw_labels: List[str] = None,
      ignore_extra_tokens: bool = False,
      ignore_start_end: bool = False,
  ):
      """
      Processes the data and returns features.
      Args:
          queries: text sequences
          tokenizer: such as AutoTokenizer
          max_seq_length: max sequence length minus 2 for [CLS] and [SEP], when -1 - use the max len from the data
          pad_label: pad value use for labels. By default, it's the neutral label.
          raw_labels: list of labels for every word in a sequence
          label_ids: dict to map labels to label ids.
              Starts with pad_label->0 and then increases in alphabetical order.
              Required for training and evaluation, not needed for inference.
          ignore_extra_tokens: whether to ignore extra tokens in the loss_mask
          ignore_start_end: whether to ignore bos and eos tokens in the loss_mask
      """
      all_subtokens = []
      all_loss_mask = []
      all_subtokens_mask = []
      all_segment_ids = []
      all_input_ids = []
      all_input_mask = []
      sent_lengths = []
      all_labels = []
      with_label = False
  
      if raw_labels is not None:
          with_label = True
  
      for i, query in enumerate(queries):
          words = query.strip().split()
  
          # add bos token
          subtokens = [tokenizer.cls_token]
          loss_mask = [1 - ignore_start_end]
          subtokens_mask = [0]
          if with_label:
              pad_id = label_ids[pad_label]
              labels = [pad_id]
              query_labels = [label_ids[lab] for lab in raw_labels[i]]
  
          for j, word in enumerate(words):
              word_tokens = tokenizer.text_to_tokens(word)
  
              # to handle emojis that could be neglected during tokenization
              if len(word.strip()) > 0 and len(word_tokens) == 0:
                  word_tokens = [
                      tokenizer.ids_to_tokens(tokenizer.unk_id)
                  ]
  
              subtokens.extend(word_tokens)
  
              loss_mask.append(1)
              loss_mask.extend(
                  [int(not ignore_extra_tokens)]
                  * (len(word_tokens) - 1)
              )
  
              subtokens_mask.append(1)
              subtokens_mask.extend([0] * (len(word_tokens) - 1))
  
              if with_label:
                  labels.extend([query_labels[j]] * len(word_tokens))
          # add eos token
          subtokens.append(tokenizer.sep_token)
          loss_mask.append(1 - ignore_start_end)
          subtokens_mask.append(0)
          sent_lengths.append(len(subtokens))
          all_subtokens.append(subtokens)
          all_loss_mask.append(loss_mask)
          all_subtokens_mask.append(subtokens_mask)
          all_input_mask.append([1] * len(subtokens))
  
          if with_label:
              labels.append(pad_id)
              all_labels.append(labels)
  
      max_seq_length_data = max(sent_lengths)
      max_seq_length = (
          min(max_seq_length, max_seq_length_data)
          if max_seq_length > 0
          else max_seq_length_data
      )
      logging.info(f"Setting Max Seq length to: {max_seq_length}")
      get_stats(sent_lengths)
      too_long_count = 0
  
      for i, subtokens in enumerate(all_subtokens):
          if len(subtokens) > max_seq_length:
              subtokens = [tokenizer.cls_token] + subtokens[
                  -max_seq_length + 1 :
              ]
              all_input_mask[i] = [1] + all_input_mask[i][
                  -max_seq_length + 1 :
              ]
              all_loss_mask[i] = [
                  int(not ignore_start_end)
              ] + all_loss_mask[i][-max_seq_length + 1 :]
              all_subtokens_mask[i] = [0] + all_subtokens_mask[i][
                  -max_seq_length + 1 :
              ]
  
              if with_label:
                  all_labels[i] = [pad_id] + all_labels[i][
                      -max_seq_length + 1 :
                  ]
              too_long_count += 1
  
          all_input_ids.append(tokenizer.tokens_to_ids(subtokens))
  
          if len(subtokens) < max_seq_length:
              extra = max_seq_length - len(subtokens)
              all_input_ids[i] = all_input_ids[i] + [0] * extra
              all_loss_mask[i] = all_loss_mask[i] + [0] * extra
              all_subtokens_mask[i] = (
                  all_subtokens_mask[i] + [0] * extra
              )
              all_input_mask[i] = all_input_mask[i] + [0] * extra
  
              if with_label:
                  all_labels[i] = all_labels[i] + [pad_id] * extra
  
          all_segment_ids.append([0] * max_seq_length)
  
      logging.warning(
          f"{too_long_count} are longer than {max_seq_length}"
      )
  
      for i in range(min(len(all_input_ids), 1)):
          logging.info("*** Example ***")
          logging.info("i: %s", i)
          logging.info(
              "subtokens: %s",
              " ".join(list(map(str, all_subtokens[i]))),
          )
          logging.info(
              "loss_mask: %s",
              " ".join(list(map(str, all_loss_mask[i]))),
          )
          logging.info(
              "input_mask: %s",
              " ".join(list(map(str, all_input_mask[i]))),
          )
          logging.info(
              "subtokens_mask: %s",
              " ".join(list(map(str, all_subtokens_mask[i]))),
          )
          if with_label:
              logging.info(
                  "labels: %s", " ".join(list(map(str, all_labels[i])))
              )
      return (
          all_input_ids,
          all_segment_ids,
          all_input_mask,
          all_subtokens_mask,
          all_loss_mask,
          all_labels,
      )
  
imports: import os
  import pickle
  import tempfile
  import time
  from typing import Dict, List, Optional
  import numpy
  from nemo.collections.common.tokenizers.tokenizer_spec import TokenizerSpec
  from nemo.collections.nlp.data.data_utils.data_preprocessing import get_stats
  from nemo.core.classes import Dataset
  from nemo.core.neural_types import ChannelType, LabelsType, MaskType, NeuralType
  from nemo.utils import logging
  from nemo.utils.get_rank import is_global_rank_zero
label_desc: Use image node_id 1 for calling a global function with example usage: get_features(queries, tokenizer, max_seq_length, label_ids, pad_label, raw_labels, ignore_extra_tokens, ignore_start_end) and returns: all_input_ids,  all_segment_ids,  all_input_mask,  all_subtokens_mask,  all_loss_mask,  all_labels