code: def _tokenize_files(
      func,
      files,
      path,
      output_dir=None,
      output_names=None,
      n_workers=defaults.cpus,
      rules=None,
      tok=None,
      encoding="utf8",
      skip_if_exists=False,
  ):
      "Tokenize text `files` in parallel using `n_workers`"
      if tok is None:
          tok = WordTokenizer()
      output_dir = Path(
          ifnone(output_dir, path.parent / f"{path.name}_tok")
      )
      if skip_if_exists and output_dir.exists():
          return output_dir
      output_dir.mkdir(exist_ok=True)
      if output_names is None:
          output_names = L(
              output_dir / f.relative_to(path) for f in files
          )
      rules = partial(Path.read_text, encoding=encoding) + L(
          ifnone(rules, defaults.text_proc_rules.copy())
      )
  
      lengths, counter = {}, Counter()
      for i, tok in parallel_tokenize(
          files, tok, rules, n_workers=n_workers
      ):
          out = func(i, output_dir)
          out.mk_write(" ".join(tok), encoding=encoding)
          lengths[str(files[i].relative_to(path))] = len(tok)
          counter.update(tok)
  
      save_pickle(output_dir / fn_lengths_pkl, lengths)
      save_pickle(output_dir / fn_counter_pkl, counter)
      return output_dir
  
imports: from __future__ import annotations
  from ..torch_basics import *
  from ..data.all import *
  import html
label_desc: Use image node_id 12 for calling a global function with example usage: _tokenize_files(func, files, path, output_dir, output_names, n_workers, rules, tok, encoding, skip_if_exists) and returns: output_dir, output_dir