class_docstr: Tokenizes the text with respect to existent phrases in the vocab.
  
  This tokenizers respects phrases that are in the vocab. Phrases are separated with 'ngram_separator', for example,
  in Google News word2vec file, ngrams are separated with a _ like New_York. These phrases are detected in text and merged as one special token. (New York is the ... => [New_York, is, the])
code: def tokenize(self, text: str) -> List[int]:
      tokens = nltk.word_tokenize(text, preserve_line=True)
  
      # phrase detection
      for ngram_len in sorted(self.ngram_lengths, reverse=True):
          idx = 0
          while idx <= len(tokens) - ngram_len:
              ngram = self.ngram_separator.join(
                  tokens[idx : idx + ngram_len]
              )
              if ngram in self.ngram_lookup:
                  tokens[idx : idx + ngram_len] = [ngram]
              elif ngram.lower() in self.ngram_lookup:
                  tokens[idx : idx + ngram_len] = [ngram.lower()]
              idx += 1
  
      # Map tokens to idx, filter stop words
      tokens_filtered = []
      for token in tokens:
          if token in self.stop_words:
              continue
          elif token in self.word2idx:
              tokens_filtered.append(self.word2idx[token])
              continue
  
          token = token.lower()
          if token in self.stop_words:
              continue
          elif token in self.word2idx:
              tokens_filtered.append(self.word2idx[token])
              continue
  
          token = token.strip(string.punctuation)
          if token in self.stop_words:
              continue
          elif len(token) > 0 and token in self.word2idx:
              tokens_filtered.append(self.word2idx[token])
              continue
  
      return tokens_filtered
  
file_path: sentence-transformers/sentence_transformers/models/tokenizer/PhraseTokenizer.py
imports: from typing import List, Iterable
  import collections
  import string
  import os
  import json
  import logging
  from .WordTokenizer import WordTokenizer, ENGLISH_STOP_WORDS
  import nltk
label_desc: Use image node_id 4 for calling the PhraseTokenizer obj's underlying member method code with example usage: obj.tokenize(text) and returns: tokens_filtered