code: def test_convert_units_file(tokenizer):
      """
      Tests reading some text from a file and converting that to units
      """
      with tempfile.TemporaryDirectory(
          dir=TEST_WORKING_DIR
      ) as test_dir:
          # two nl test case, read from file
          labels = "00000000000000000001\n\n000\n\n"
          raw_text = "This is a      test.\n\nfoo\n\n"
          txt_file, label_file = write_tokenizer_input(
              test_dir, raw_text, labels
          )
  
          batches = DataLoader(
              tokenizer.config,
              input_files={"txt": txt_file, "label": label_file},
              vocab=tokenizer.vocab,
              evaluation=True,
              dictionary=tokenizer.trainer.dictionary,
          )
          assert batches.data == EXPECTED_TWO_NL_FILE
          check_labels(batches.labels(), EXPECTED_TWO_NL_FILE_LABELS)
  
          # one nl test case, read from file
          labels = "000000000000000000010000\n\n"
          raw_text = "This is a      test.\nfoo\n\n"
          txt_file, label_file = write_tokenizer_input(
              test_dir, raw_text, labels
          )
  
          batches = DataLoader(
              tokenizer.config,
              input_files={"txt": txt_file, "label": label_file},
              vocab=tokenizer.vocab,
              evaluation=True,
              dictionary=tokenizer.trainer.dictionary,
          )
          assert batches.data == EXPECTED_ONE_NL_FILE
          check_labels(batches.labels(), EXPECTED_ONE_NL_FILE_LABELS)
  
          skip_newline_config = dict(tokenizer.config)
          skip_newline_config["skip_newline"] = True
          labels = "000000000000000000010000\n\n"
          raw_text = "This is a      test.\nfoo\n\n"
          txt_file, label_file = write_tokenizer_input(
              test_dir, raw_text, labels
          )
  
          batches = DataLoader(
              skip_newline_config,
              input_files={"txt": txt_file, "label": label_file},
              vocab=tokenizer.vocab,
              evaluation=True,
              dictionary=tokenizer.trainer.dictionary,
          )
          assert batches.data == EXPECTED_SKIP_NL_FILE
          check_labels(batches.labels(), EXPECTED_SKIP_NL_FILE_LABELS)
  
file_path: stanza/stanza/tests/tokenization/test_tokenize_data.py
imports: import pytest
  import tempfile
  import numpy
  import stanza
  from stanza import Pipeline
  from stanza.tests import *
  from stanza.models.tokenization.data import DataLoader, NUMERIC_RE
label_desc: Use image node_id 7 for calling a global function with example usage: test_convert_units_file(tokenizer) without return types