code: def test_featurization_of_task_data():
      # Test reading the task-specific toy dataset into examples.
      task = create_task_from_config_path(
          os.path.join(
              os.path.dirname(__file__), "resources/mnli.json"
          ),
          verbose=False,
      )
      # Test getting train, val, and test examples. Only the contents of train are checked.
      train_examples = task.get_train_examples()
      val_examples = task.get_val_examples()
      test_examples = task.get_test_examples()
      for train_example_dataclass, raw_example_dict in zip(
          train_examples, TRAIN_EXAMPLES
      ):
          assert train_example_dataclass.to_dict() == raw_example_dict
      assert val_examples
      assert test_examples
  
      # Testing conversion of examples into tokenized examples
      # the dummy tokenizer requires a vocab — using a Counter here to find that vocab from the data:
      token_counter = Counter()
      for example in train_examples:
          token_counter.update(example.premise.split())
          token_counter.update(example.hypothesis.split())
      token_vocab = list(token_counter.keys())
      tokenizer = SimpleSpaceTokenizer(vocabulary=token_vocab)
      tokenized_examples = [
          example.tokenize(tokenizer) for example in train_examples
      ]
      for tokenized_example, expected_tokenized_example in zip(
          tokenized_examples, TOKENIZED_TRAIN_EXAMPLES
      ):
          assert (
              tokenized_example.to_dict() == expected_tokenized_example
          )
  
      # Testing conversion of a tokenized example to a featurized example
      train_example_0_length = len(tokenized_examples[0].premise) + len(
          tokenized_examples[0].hypothesis
      )
      feat_spec = (
          JiantTransformersModelFactory.build_featurization_spec(
              model_type=ModelArchitectures.BERT.value,
              max_seq_length=train_example_0_length,
          )
      )
      featurized_examples = [
          tokenized_example.featurize(
              tokenizer=tokenizer, feat_spec=feat_spec
          )
          for tokenized_example in tokenized_examples
      ]
      featurized_example_0_dict = featurized_examples[0].to_dict()
      # not bothering to compare the input_ids because they were made by a dummy tokenizer.
      assert "input_ids" in featurized_example_0_dict
      assert (
          featurized_example_0_dict["guid"]
          == FEATURIZED_TRAIN_EXAMPLE_0["guid"]
      )
      assert (
          featurized_example_0_dict["input_mask"]
          == FEATURIZED_TRAIN_EXAMPLE_0["input_mask"]
      ).all()
      assert (
          featurized_example_0_dict["segment_ids"]
          == FEATURIZED_TRAIN_EXAMPLE_0["segment_ids"]
      ).all()
      assert (
          featurized_example_0_dict["label_id"]
          == FEATURIZED_TRAIN_EXAMPLE_0["label_id"]
      )
      assert (
          featurized_example_0_dict["tokens"]
          == FEATURIZED_TRAIN_EXAMPLE_0["tokens"]
      )
  
file_path: jiant/tests/tasks/lib/test_mnli.py
imports: import numpy
  import os
  from collections import Counter
  from jiant.shared import model_resolution
  from jiant.shared.model_resolution import ModelArchitectures
  from jiant.tasks.retrieval import create_task_from_config_path
  from jiant.utils.testing.tokenizer import SimpleSpaceTokenizer
  from jiant.proj.main.modeling.primary import JiantTransformersModelFactory
label_desc: Use image node_id 1 for calling a global function with example usage: test_featurization_of_task_data() without return types