class_docstr: Creates a dataset to use for the task of joint intent
  and slot classification with pretrained model.
  
  For a dataset to use during inference without labels, see
  IntentSlotDataset.
class_docstr_tok: ["Creates","a","dataset","to","use","for","the","task","of","joint","intent","and","slot","classification","with","pretrained","model",".","For","a","dataset","to","use","during","inference","without","labels",",","see","IntentSlotDataset","."]
docstr: Convert queries (utterance, intent label and slot labels) to BERT input format 
docstr_tok: ["Convert","queries","(","utterance",",","intent","label","and","slot","labels",")","to","BERT","input","format"]
code: def get_features(
      queries,
      max_seq_length,
      tokenizer,
      pad_label=128,
      word_level_slots=None,
      ignore_extra_tokens=False,
      ignore_start_end=False,
  ):
      """
      Convert queries (utterance, intent label and slot labels) to BERT input format
      """
  
      all_subtokens = []
      all_loss_mask = []
      all_subtokens_mask = []
      all_segment_ids = []
      all_input_ids = []
      all_input_mask = []
      sent_lengths = []
      all_slots = []
  
      with_label = word_level_slots is not None
  
      for i, query in enumerate(queries):
          words = query.strip().split()
          subtokens = [tokenizer.cls_token]
          loss_mask = [1 - ignore_start_end]
          subtokens_mask = [0]
          if with_label:
              slots = [pad_label]
  
          for j, word in enumerate(words):
              word_tokens = tokenizer.text_to_tokens(word)
  
              # to handle emojis that could be neglected during tokenization
              if len(word.strip()) > 0 and len(word_tokens) == 0:
                  word_tokens = [
                      tokenizer.ids_to_tokens(tokenizer.unk_id)
                  ]
  
              subtokens.extend(word_tokens)
              # mask all sub-word tokens except the first token in a word
              # use the label for the first sub-word token as the label for the entire word to eliminate need for disambiguation
              loss_mask.append(1)
              loss_mask.extend(
                  [int(not ignore_extra_tokens)]
                  * (len(word_tokens) - 1)
              )
  
              subtokens_mask.append(1)
              subtokens_mask.extend([0] * (len(word_tokens) - 1))
  
              if with_label:
                  slots.extend(
                      [word_level_slots[i][j]] * len(word_tokens)
                  )
  
          subtokens.append(tokenizer.sep_token)
          loss_mask.append(1 - ignore_start_end)
          subtokens_mask.append(0)
          sent_lengths.append(len(subtokens))
          all_subtokens.append(subtokens)
          all_loss_mask.append(loss_mask)
          all_subtokens_mask.append(subtokens_mask)
          all_input_mask.append([1] * len(subtokens))
          if with_label:
              slots.append(pad_label)
              all_slots.append(slots)
      max_seq_length_data = max(sent_lengths)
      max_seq_length = (
          min(max_seq_length, max_seq_length_data)
          if max_seq_length > 0
          else max_seq_length_data
      )
      logging.info(f"Setting max length to: {max_seq_length}")
      get_stats(sent_lengths)
  
      # truncate and pad samples
      (
          all_slots,
          all_subtokens,
          all_input_mask,
          all_loss_mask,
          all_subtokens_mask,
          all_input_ids,
          all_segment_ids,
      ) = DialogueBERTDataset.truncate_and_pad(
          max_seq_length,
          ignore_start_end,
          with_label,
          pad_label,
          tokenizer,
          all_slots,
          all_subtokens,
          all_input_mask,
          all_loss_mask,
          all_subtokens_mask,
          all_input_ids,
          all_segment_ids,
      )
  
      # log examples for debugging
      logging.debug("*** Some Examples of Processed Data ***")
      for i in range(min(len(all_input_ids), 5)):
          logging.debug("i: %s" % (i))
          logging.debug(
              "subtokens: %s"
              % " ".join(list(map(str, all_subtokens[i])))
          )
          logging.debug(
              "loss_mask: %s"
              % " ".join(list(map(str, all_loss_mask[i])))
          )
          logging.debug(
              "input_mask: %s"
              % " ".join(list(map(str, all_input_mask[i])))
          )
          logging.debug(
              "subtokens_mask: %s"
              % " ".join(list(map(str, all_subtokens_mask[i])))
          )
          if with_label:
              logging.debug(
                  "slots_label: %s"
                  % " ".join(list(map(str, all_slots[i])))
              )
  
      return (
          all_input_ids,
          all_segment_ids,
          all_input_mask,
          all_loss_mask,
          all_subtokens_mask,
          all_slots,
      )
  
imports: from typing import Dict, Optional
  import numpy
  from nemo.collections.nlp.data.data_utils import get_stats
  from nemo.collections.nlp.data.dialogue.dataset.dialogue_dataset import DialogueDataset
  from nemo.core.neural_types import ChannelType, LabelsType, MaskType, NeuralType
  from nemo.utils import logging
label_desc: Use image node_id 7 for calling the DialogueBERTDataset obj's underlying member method code with example usage: obj.get_features(queries, max_seq_length, tokenizer, pad_label, word_level_slots, ignore_extra_tokens, ignore_start_end) and returns: all_input_ids,  all_segment_ids,  all_input_mask,  all_loss_mask,  all_subtokens_mask,  all_slots