code: def test_convert_units_raw_text(tokenizer):
      """
      Tests converting a couple small segments to units
      """
      raw_text = "This is a      test\n\nfoo"
      batches = DataLoader(
          tokenizer.config,
          input_text=raw_text,
          vocab=tokenizer.vocab,
          evaluation=True,
          dictionary=tokenizer.trainer.dictionary,
      )
      assert batches.data == EXPECTED_TWO_NL_RAW
  
      raw_text = "This is a      test\nfoo"
      batches = DataLoader(
          tokenizer.config,
          input_text=raw_text,
          vocab=tokenizer.vocab,
          evaluation=True,
          dictionary=tokenizer.trainer.dictionary,
      )
      assert batches.data == EXPECTED_ONE_NL_RAW
  
      skip_newline_config = dict(tokenizer.config)
      skip_newline_config["skip_newline"] = True
      batches = DataLoader(
          skip_newline_config,
          input_text=raw_text,
          vocab=tokenizer.vocab,
          evaluation=True,
          dictionary=tokenizer.trainer.dictionary,
      )
      assert batches.data == EXPECTED_SKIP_NL_RAW
  
file_path: stanza/stanza/tests/tokenization/test_tokenize_data.py
imports: import pytest
  import tempfile
  import numpy
  import stanza
  from stanza import Pipeline
  from stanza.tests import *
  from stanza.models.tokenization.data import DataLoader, NUMERIC_RE
label_desc: Use image node_id 5 for calling a global function with example usage: test_convert_units_raw_text(tokenizer) without return types