code: def test_create_vocabulary_idf(compute_idf: bool):
      data = pd.DataFrame(
          [
              "Hello, I'm a single sentence!",
              "And another sentence",
              "And the very very last one",
          ]
      )
      column = data[0]
      preprocessing_parameters = TextPreprocessingConfig().to_dict()
  
      vocabulary = strings_utils.create_vocabulary(
          column,
          tokenizer_type=preprocessing_parameters["tokenizer"],
          num_most_frequent=preprocessing_parameters["most_common"],
          lowercase=preprocessing_parameters["lowercase"],
          vocab_file=preprocessing_parameters["vocab_file"],
          unknown_symbol=preprocessing_parameters["unknown_symbol"],
          padding_symbol=preprocessing_parameters["padding_symbol"],
          pretrained_model_name_or_path=preprocessing_parameters[
              "pretrained_model_name_or_path"
          ],
          compute_idf=compute_idf,
          add_special_symbols=False,
      )
  
      str2idf = vocabulary.str2idf
  
      if not compute_idf:
          assert str2idf is None
          return
  
      idf2str = defaultdict(set)
      for k, v in str2idf.items():
          idf2str[v].add(k)
      idf_sorted = sorted(idf2str.items(), key=lambda x: x[0])
      assert len(idf_sorted) == 3
  
      # Unknown symbol should have the lowest idf as it never appears in any documents
      assert idf_sorted[0][0] == 0
      assert idf_sorted[0][1] == {"<UNK>"}
  
      # "sentence" and "and" should be next, as they appear in two docs each
      assert idf_sorted[1][0] > idf_sorted[0][0]
      assert idf_sorted[1][1] == {"sentence", "And"}
  
      # finally, every token that only appears once
      assert idf_sorted[2][0] > idf_sorted[1][0]
      assert idf_sorted[2][1] == {
          ",",
          "I",
          "'",
          "one",
          "very",
          "single",
          "the",
          "m",
          "!",
          "last",
          "Hello",
          "a",
          "another",
      }
  
file_path: ludwig/tests/ludwig/utils/test_strings_utils.py
imports: from collections import defaultdict
  import numpy
  import pandas
  import pytest
  from ludwig.schema.features.preprocessing.text import TextPreprocessingConfig
  from ludwig.utils import strings_utils
label_desc: Use image node_id 13 for calling a global function with example usage: test_create_vocabulary_idf(compute_idf) without return types