code: def get_features(
      queries,
      max_seq_length,
      tokenizer,
      pad_label=128,
      raw_slots=None,
      ignore_extra_tokens=False,
      ignore_start_end=False,
  ):
      all_subtokens = []
      all_loss_mask = []
      all_subtokens_mask = []
      all_segment_ids = []
      all_input_ids = []
      all_input_mask = []
      sent_lengths = []
      all_slots = []
  
      with_label = False
      if raw_slots is not None:
          with_label = True
  
      for i, query in enumerate(queries):
          words = query.strip().split()
          subtokens = [tokenizer.cls_token]
          loss_mask = [1 - ignore_start_end]
          subtokens_mask = [0]
          if with_label:
              slots = [pad_label]
  
          for j, word in enumerate(words):
              word_tokens = tokenizer.text_to_tokens(word)
  
              # to handle emojis that could be neglected during tokenization
              if len(word.strip()) > 0 and len(word_tokens) == 0:
                  word_tokens = [
                      tokenizer.ids_to_tokens(tokenizer.unk_id)
                  ]
  
              subtokens.extend(word_tokens)
  
              loss_mask.append(1)
              loss_mask.extend(
                  [int(not ignore_extra_tokens)]
                  * (len(word_tokens) - 1)
              )
  
              subtokens_mask.append(1)
              subtokens_mask.extend([0] * (len(word_tokens) - 1))
  
              if with_label:
                  slots.extend([raw_slots[i][j]] * len(word_tokens))
  
          subtokens.append(tokenizer.sep_token)
          loss_mask.append(1 - ignore_start_end)
          subtokens_mask.append(0)
          sent_lengths.append(len(subtokens))
          all_subtokens.append(subtokens)
          all_loss_mask.append(loss_mask)
          all_subtokens_mask.append(subtokens_mask)
          all_input_mask.append([1] * len(subtokens))
          if with_label:
              slots.append(pad_label)
              all_slots.append(slots)
  
      max_seq_length_data = max(sent_lengths)
      max_seq_length = (
          min(max_seq_length, max_seq_length_data)
          if max_seq_length > 0
          else max_seq_length_data
      )
      logging.info(f"Setting max length to: {max_seq_length}")
      get_stats(sent_lengths)
      too_long_count = 0
  
      for i, subtokens in enumerate(all_subtokens):
          if len(subtokens) > max_seq_length:
              subtokens = [tokenizer.cls_token] + subtokens[
                  -max_seq_length + 1 :
              ]
              all_input_mask[i] = [1] + all_input_mask[i][
                  -max_seq_length + 1 :
              ]
              all_loss_mask[i] = [1 - ignore_start_end] + all_loss_mask[
                  i
              ][-max_seq_length + 1 :]
              all_subtokens_mask[i] = [0] + all_subtokens_mask[i][
                  -max_seq_length + 1 :
              ]
  
              if with_label:
                  all_slots[i] = [pad_label] + all_slots[i][
                      -max_seq_length + 1 :
                  ]
              too_long_count += 1
  
          all_input_ids.append(
              [tokenizer.tokens_to_ids(t) for t in subtokens]
          )
  
          if len(subtokens) < max_seq_length:
              extra = max_seq_length - len(subtokens)
              all_input_ids[i] = all_input_ids[i] + [0] * extra
              all_loss_mask[i] = all_loss_mask[i] + [0] * extra
              all_subtokens_mask[i] = (
                  all_subtokens_mask[i] + [0] * extra
              )
              all_input_mask[i] = all_input_mask[i] + [0] * extra
  
              if with_label:
                  all_slots[i] = all_slots[i] + [pad_label] * extra
  
          all_segment_ids.append([0] * max_seq_length)
  
      logging.info(f"{too_long_count} are longer than {max_seq_length}")
  
      # May be useful for debugging
      logging.debug("*** Some Examples of Processed Data ***")
      for i in range(min(len(all_input_ids), 5)):
          logging.debug("i: %s" % (i))
          logging.debug(
              "subtokens: %s"
              % " ".join(list(map(str, all_subtokens[i])))
          )
          logging.debug(
              "loss_mask: %s"
              % " ".join(list(map(str, all_loss_mask[i])))
          )
          logging.debug(
              "input_mask: %s"
              % " ".join(list(map(str, all_input_mask[i])))
          )
          logging.debug(
              "subtokens_mask: %s"
              % " ".join(list(map(str, all_subtokens_mask[i])))
          )
          if with_label:
              logging.debug(
                  "slots_label: %s"
                  % " ".join(list(map(str, all_slots[i])))
              )
  
      return (
          all_input_ids,
          all_segment_ids,
          all_input_mask,
          all_loss_mask,
          all_subtokens_mask,
          all_slots,
      )
  
file_path: NeMo/nemo/collections/nlp/data/intent_slot_classification/intent_slot_classification_dataset.py
imports: from typing import Dict, Optional
  import numpy
  from nemo.collections.common.tokenizers.tokenizer_spec import TokenizerSpec
  from nemo.collections.nlp.data.data_utils import get_stats
  from nemo.core.classes import Dataset
  from nemo.core.neural_types import ChannelType, LabelsType, MaskType, NeuralType
  from nemo.utils import logging
label_desc: Use image node_id 1 for calling a global function with example usage: get_features(queries, max_seq_length, tokenizer, pad_label, raw_slots, ignore_extra_tokens, ignore_start_end) and returns: all_input_ids,  all_segment_ids,  all_input_mask,  all_loss_mask,  all_subtokens_mask,  all_slots