code: def create_vocabulary(
      data: Series,
      tokenizer_type: str = "space",
      lowercase: bool = True,
      num_most_frequent: int = None,
      vocab_file: str = None,
      add_special_symbols: bool = True,
      unknown_symbol: str = UNKNOWN_SYMBOL,
      padding_symbol: str = PADDING_SYMBOL,
      start_symbol: str = START_SYMBOL,
      stop_symbol: str = STOP_SYMBOL,
      pretrained_model_name_or_path: str = None,
      ngram_size: Optional[int] = None,
      compute_idf: bool = False,
      processor: DataFrameEngine = PANDAS,
      prompt_template: str = "",
  ) -> Vocabulary:
      """Computes a vocabulary over the provided data frame.
  
      This function is used when the data consists of multiple tokens within one example. E.g., words in a text feature,
      items in a set feature, etc. If the feature only contains a single token like for category features,
      `create_vocabulary_single_token` should be used instead, as it is more efficient.
  
      A tokenizer is specified using the `tokenizer_type`. The tokenizer will be used to process all of the data
      provided, producing an indexed vocabulary with frequency counts. If the `tokenizer_type` is 'hf_tokenizer',
      then a pre-trained huggingface tokenizer is loaded from `pretrained_model_name_or_path` and that vocabulary is
      used directly.
  
      The UNKNOWN special symbol is always included in the final vocabulary. Additional special symbols (PADDING, START,
      STOP) are added if add_special_symbols=True. If the tokenizer is a pre-trained huggingface tokenizer, then the
      special symbols are taken from the tokenizer's vocabulary.
  
      Args:
          prompt_template: The prompt template for the model. Applicable only to LLMs.
          data: Series of string data.
          tokenizer_type: Tokenizer type. Can be a tokenizer registry value or 'hf_tokenizer' for huggingface.
          lowercase: Whether to lowercase all strings.
          num_most_frequent: Upper limit on vocabulary size.,
          add_special_symbols: If True, START, STOP, PADDING special symbols are added to the vocabulary. UNKNOWN is
              always added.
          unknown_symbol: String representation for the UNKNOWN symbol.
          padding_symbol: String representation for the PADDING symbol.
          start_symbol: String representation for the START symbol.
          stop_symbol: String representation for the STOP symbol.
          pretrained_model_name_or_path: Name/path to huggingface model.
          ngram_size: Size of the n-gram when using `ngram` tokenizer.
          compute_idf: If True, computes the inverse document frequency for each token.
          processor: Which processor to use to process data.
  
      Returns:
          Vocabulary object containing metadata about the vocab.
  
      TODO(Justin): Clean up pad_idx, padding_symbol, unknown_symbol return, as no one seems to be using it.
      """
      tokenizer = get_tokenizer_from_registry(tokenizer_type)(
          vocab_file=vocab_file,
          pretrained_model_name_or_path=pretrained_model_name_or_path,
          ngram_size=ngram_size,
      )
  
      # Number of tokens in template.
      prompt_template_num_tokens = -1
      if prompt_template:
          prompt_without_bracketed_elements = remove_bracketed_elements(
              prompt_template
          )
          prompt_template_num_tokens = len(
              tokenizer(prompt_without_bracketed_elements)
          )
  
      # Tokenize the data.
      def process_line(line):
          return tokenizer(line.lower() if lowercase else line)
  
      processed_lines = processor.map_objects(data, process_line)
      processed_counts = processed_lines.explode().value_counts(
          sort=False
      )
      processed_counts = processor.compute(processed_counts)
      unit_counts = Counter(dict(processed_counts))
      max_sequence_length = processor.compute(
          processed_lines.map(len).max()
      )
      sequence_length_99ptile = processor.compute(
          processed_lines.map(len).quantile(0.99)
      )
  
      if tokenizer_type != "hf_tokenizer":
          # For non-HF tokenizers, add 2 for start and stop symbols.
          max_sequence_length += 2
          sequence_length_99ptile += 2
  
      pad_idx = None
      if tokenizer_type == "hf_tokenizer":
          # Replace the special symbols with the ones from the tokenizer.
          unknown_symbol = tokenizer.get_unk_token()
          padding_symbol = tokenizer.get_pad_token()
          pad_idx = tokenizer.convert_token_to_id(padding_symbol)
  
      vocab: List[str] = _get_vocabulary(
          tokenizer_type,
          tokenizer,
          vocab_file,
          unknown_symbol,
          add_special_symbols,
          padding_symbol,
          unit_counts,
          num_most_frequent,
      )
      vocab_set = set(vocab)
  
      doc_unit_counts = None
      if compute_idf:
          # The document frequency used for TF-IDF. Similar to unit_counts, but de-duped by document.
          document_counts = (
              processed_lines.map(lambda x: set(x))
              .explode()
              .value_counts(sort=False)
          )
          document_counts = processor.compute(document_counts)
          doc_unit_counts = Counter(dict(document_counts))
  
      if tokenizer_type != "hf_tokenizer":
          if add_special_symbols:
              add_or_move_symbol(
                  vocab,
                  vocab_set,
                  stop_symbol,
                  SpecialSymbol.STOP.value,
              )
              add_or_move_symbol(
                  vocab,
                  vocab_set,
                  start_symbol,
                  SpecialSymbol.START.value,
              )
              add_or_move_symbol(
                  vocab,
                  vocab_set,
                  padding_symbol,
                  SpecialSymbol.PADDING.value,
              )
          # Always add the UNKNOWN symbol if we're using our own tokenizer.
          add_or_move_symbol(
              vocab,
              vocab_set,
              unknown_symbol,
              SpecialSymbol.UNKNOWN.value,
          )
  
      str2idx = {unit: i for i, unit in enumerate(vocab)}
      str2freq = {
          unit: unit_counts.get(unit) if unit in unit_counts else 0
          for unit in vocab
      }
      str2idf = (
          {
              unit: np.log(len(vocab) / (1 + doc_unit_counts.get(unit)))
              if unit in doc_unit_counts
              else 0
              for unit in vocab
          }
          if compute_idf
          else None
      )
  
      if pad_idx is None and padding_symbol in str2idx.keys():
          pad_idx = str2idx[padding_symbol]
  
      return Vocabulary(
          vocab=vocab,
          str2idx=str2idx,
          str2freq=str2freq,
          str2idf=str2idf,
          max_sequence_length=max_sequence_length,
          sequence_length_99ptile=sequence_length_99ptile,
          pad_idx=pad_idx,
          padding_symbol=padding_symbol,
          unknown_symbol=unknown_symbol,
          prompt_template_num_tokens=prompt_template_num_tokens,
      )
  
imports: import logging
  import re
  import unicodedata
  from collections import Counter
  from dataclasses import dataclass
  from enum import Enum
  from typing import Dict, List, Optional, Set, Union
  import numpy
  from dateutil.parser import parse
  from ludwig.constants import PADDING_SYMBOL, START_SYMBOL, STOP_SYMBOL, UNKNOWN_SYMBOL
  from ludwig.data.dataframe.base import DataFrameEngine
  from ludwig.data.dataframe.pandas import PANDAS
  from ludwig.utils.fs_utils import open_file
  from ludwig.utils.math_utils import int_type
  from ludwig.utils.tokenizers import get_tokenizer_from_registry
  from ludwig.utils.types import Series
label_desc: Use image node_id 20 for calling a global function with example usage: create_vocabulary(data, tokenizer_type, lowercase, num_most_frequent, vocab_file, add_special_symbols, unknown_symbol, padding_symbol, start_symbol, stop_symbol, pretrained_model_name_or_path, ngram_size, compute_idf, processor, prompt_template) and returns: Vocabulary, tokenizer