code: def ingest_tokens(
      opts, transforms, n_sample, learner, stride, offset
  ):
      def _mp_ingest(data):
          func = partial(process, CorpusName.TRAIN)
          chunk = len(data) // opts.num_threads
          with mp.Pool(opts.num_threads) as pool:
              buckets = pool.map(
                  func,
                  [
                      data[i * chunk : (i + 1) * chunk]
                      for i in range(0, opts.num_threads)
                  ],
              )
          for bucket in buckets:
              for ex in bucket:
                  if ex is not None:
                      src_line, tgt_line = (
                          ex["src"]["src"],
                          ex["tgt"]["tgt"],
                      )
                      learner.ingest(src_line)
                      learner.ingest(tgt_line)
  
      corpora = get_corpora(opts, task=CorpusTask.TRAIN)
      datasets_iterables = build_corpora_iters(
          corpora,
          transforms,
          opts.data,
          skip_empty_level=opts.skip_empty_level,
          stride=stride,
          offset=offset,
      )
      to_ingest = []
      for c_name, c_iter in datasets_iterables.items():
          for i, item in enumerate(c_iter):
              if n_sample >= 0 and i >= n_sample:
                  break
              if len(to_ingest) >= MAXBUCKETSIZE:
                  _mp_ingest(to_ingest)
                  to_ingest = []
              to_ingest.append(item)
          _mp_ingest(to_ingest)
  
imports: import os
  import copy
  import multiprocessing
  import pyonmttok
  from functools import partial
  from onmt.utils.logging import init_logger, logger
  from onmt.utils.misc import set_random_seed, check_path
  from onmt.utils.parse import ArgumentParser
  from onmt.opts import data_prepare_opts
  from onmt.inputters.text_corpus import build_corpora_iters, get_corpora
  from onmt.inputters.text_utils import process, append_features_to_text
  from onmt.transforms import make_transforms, get_transforms_cls
  from onmt.constants import CorpusName, CorpusTask
  from collections import Counter
label_desc: Use image node_id 5 for calling a global function with example usage: ingest_tokens(opts, transforms, n_sample, learner, stride, offset) without return types