class_docstr: Tokenizes the text with respect to existent phrases in the vocab.
  
  This tokenizers respects phrases that are in the vocab. Phrases are separated with 'ngram_separator', for example,
  in Google News word2vec file, ngrams are separated with a _ like New_York. These phrases are detected in text and merged as one special token. (New York is the ... => [New_York, is, the])
code: def set_vocab(self, vocab: Iterable[str]):
      self.vocab = vocab
      self.word2idx = collections.OrderedDict(
          [(word, idx) for idx, word in enumerate(vocab)]
      )
  
      # Check for ngram in vocab
      self.ngram_lookup = set()
      self.ngram_lengths = set()
      for word in vocab:
          if (
              self.ngram_separator is not None
              and self.ngram_separator in word
          ):
              # Sum words might me malformed in e.g. google news word2vec, containing two or more _ after each other
              ngram_count = word.count(self.ngram_separator) + 1
              if (
                  self.ngram_separator + self.ngram_separator
                  not in word
                  and ngram_count <= self.max_ngram_length
              ):
                  self.ngram_lookup.add(word)
                  self.ngram_lengths.add(ngram_count)
  
      if len(vocab) > 0:
          logger.info(
              "PhraseTokenizer - Phrase ngram lengths: {}".format(
                  self.ngram_lengths
              )
          )
          logger.info(
              "PhraseTokenizer - Num phrases: {}".format(
                  len(self.ngram_lookup)
              )
          )
  
file_path: sentence-transformers/sentence_transformers/models/tokenizer/PhraseTokenizer.py
imports: from typing import List, Iterable
  import collections
  import string
  import os
  import json
  import logging
  from .WordTokenizer import WordTokenizer, ENGLISH_STOP_WORDS
  import nltk
label_desc: Use image node_id 3 for calling the PhraseTokenizer obj's underlying member method code with example usage: obj.set_vocab(vocab) without return types