code: def testBuildVocabulary(self):
      tokens = """
              your bones don't break mine do that's clear your cells react to
              bacteria and viruses differently than mine you don't get sick
              i do that's also clear but for some reason you and i react the
              exact same way to water we swallow it too fast we choke we get
              some in our lungs we drown however unreal it may seem we are
              connected you and i we're on the same curve just on opposite ends
          """.split()
      builder = utils.VocabBuilder()
      builder.add_all(tokens)
      vocab = builder.make_vocab()
      self.assertEqual(54, len(vocab))
  
      indices = vocab.lookup_all(["can i get a coffee".split()])
      self.assertEqual([[0, 21, 19, 0, 0]], indices)
      indices = vocab.lookup_all_internal(
          ["your unk unk unk unk unk unk unk unk unk".split()]
      )
      self.assertEqual(0.9, indices[1] / indices[2])
      indices = vocab.lookup_all_internal(
          [
              ["bones unk unk unk unk unk".split()],
              ["bones on on on".split()],
          ]
      )
      self.assertEqual(0.5, indices[1] / indices[2])
      indices = vocab.lookup_all_internal(
          [
              ["bones unk unk unk unk".split()],
              ["unk unk unk unk unk".split()],
          ]
      )
      self.assertEqual(0.9, indices[1] / indices[2])
  
      builder.truncate_to_vocab_size(min_counts=2)
      self.assertEqual(len(builder._counter), 18)
      builder.truncate_to_vocab_size(vocab_size=5)
      self.assertEqual(len(builder._counter), 5)
  
file_path: pytext/pytext/data/test/utils_test.py
imports: import unittest
  from pytext.data import utils
label_desc: Use image node_id 1 for calling the VocabularyTest obj's underlying member method code with example usage: obj.testBuildVocabulary() without return types