code: def _construct_tokenizer(raw_datasets, cfg_data, known_tokens=[]):
      """The actual generation instructions for a new tokenizer. Might make this more scriptable in the future...
  
      Follows closely along with https://huggingface.co/course/chapter6
      """
      try:
          len_dataset = len(raw_datasets)
  
          def batch_iterator(batch_size=4096):
              for i in range(0, len_dataset, batch_size):
                  yield raw_datasets[i : i + batch_size]["text"]
  
      except TypeError:
          # streaming dataset
          len_dataset = int(cfg_data.max_entries_in_dataset)
  
          def batch_iterator():
              for entry in iter(raw_datasets):
                  yield entry["text"]
  
      special_token_args = _get_sane_token_args()
      normalizer_sequence = _get_sane_normalizers(**cfg_data.normalizer)
      # Outline tokenizer rules:
      if (
          cfg_data.tokenizer == "Unigram"
      ):  # without the sentencepice part
          tokenizer = Tokenizer(models.Unigram())
          tokenizer.add_tokens(known_tokens)
          tokenizer.normalizer = normalizer_sequence
          tokenizer.pre_tokenizer = pre_tokenizers.Whitespace()
          # tokenizer.decoder = None
          special_tokens = list(
              set(v for k, v in special_token_args.items())
          )
  
          trainer = trainers.UnigramTrainer(
              vocab_size=cfg_data.vocab_size,
              special_tokens=special_tokens,
              unk_token=special_token_args["unk_token"],
          )
      elif cfg_data.tokenizer == "BPE":
          tokenizer = Tokenizer(models.BPE())
          tokenizer.add_tokens(known_tokens)
  
          tokenizer.normalizer = normalizer_sequence
          tokenizer.pre_tokenizer = pre_tokenizers.ByteLevel(
              add_prefix_space=True
          )
          tokenizer.decoder = decoders.ByteLevel()
  
          trainer = trainers.BpeTrainer(
              vocab_size=cfg_data.vocab_size,
              min_frequency=2,
              special_tokens=list(set(special_token_args.values())),
          )
      elif cfg_data.tokenizer == "WordPiece":
          tokenizer = Tokenizer(
              models.WordPiece(
                  unk_token=special_token_args["unk_token"]
              )
          )
          tokenizer.add_tokens(known_tokens)
  
          tokenizer.normalizer = normalizer_sequence
          tokenizer.pre_tokenizer = pre_tokenizers.Whitespace()
          tokenizer.decoder = decoders.WordPiece(prefix="##")
  
          trainer = trainers.WordPieceTrainer(
              vocab_size=cfg_data.vocab_size,
              special_tokens=list(set(special_token_args.values())),
          )
      elif cfg_data.tokenizer == "WordPieceBERT":
          # Sanity check tokenizer
          tokenizer = Tokenizer(models.WordPiece(unk_token="<unk>"))
          tokenizer.add_tokens(known_tokens)
          tokenizer.normalizer = normalizers.BertNormalizer()
          tokenizer.pre_tokenizer = pre_tokenizers.BertPreTokenizer()
          tokenizer.decoder = decoders.WordPiece(prefix="##")
  
          trainer = trainers.WordPieceTrainer(
              vocab_size=cfg_data.vocab_size,
              special_tokens=list(set(special_token_args.values())),
          )
      elif cfg_data.tokenizer == "WordLevel":
          tokenizer = Tokenizer(
              models.WordLevel(
                  unk_token=special_token_args["unk_token"]
              )
          )
          tokenizer.add_tokens(known_tokens)
          tokenizer.normalizer = normalizer_sequence
          tokenizer.pre_tokenizer = pre_tokenizers.Whitespace()
          trainer = trainers.WordLevelTrainer(
              vocab_size=cfg_data.vocab_size,
              special_tokens=list(set(special_token_args.values())),
          )
      elif cfg_data.tokenizer == "SentencePieceBPE":
          """ref https://github.com/huggingface/tokenizers/blob/main/bindings/python/py_src/tokenizers/implementations/sentencepiece_bpe.py"""
          tokenizer = Tokenizer(models.BPE())
          tokenizer.add_tokens(known_tokens)
  
          tokenizer.normalizer = normalizer_sequence
          tokenizer.pre_tokenizer = pre_tokenizers.Metaspace(
              replacement="▁", add_prefix_space=True
          )
          tokenizer.decoder = decoders.Metaspace(
              replacement="▁", add_prefix_space=True
          )
  
          trainer = trainers.BpeTrainer(
              vocab_size=cfg_data.vocab_size,
              min_frequency=2,
              special_tokens=list(set(special_token_args.values())),
          )
      elif cfg_data.tokenizer == "SentencePieceUnigram":
          tokenizer = Tokenizer(models.Unigram())
          tokenizer.add_tokens(known_tokens)
          tokenizer.normalizer = normalizer_sequence
          tokenizer.pre_tokenizer = pre_tokenizers.Metaspace(
              replacement="▁", add_prefix_space=True
          )
          tokenizer.decoder = decoders.Metaspace(
              replacement="▁", add_prefix_space=True
          )
          special_tokens = list(
              set(v for k, v in special_token_args.items())
          )
  
          trainer = trainers.UnigramTrainer(
              vocab_size=cfg_data.vocab_size,
              special_tokens=special_tokens,
              unk_token=special_token_args["unk_token"],
          )
      else:
          raise ValueError(
              f"Invalid tokenization strategy {cfg_data.tokenizer} given."
          )
  
      # Construct tokenizer
      tokenizer.train_from_iterator(
          batch_iterator(), trainer=trainer, length=len_dataset
      )
  
      if tokenizer.get_vocab_size() != cfg_data.vocab_size:
          raise RuntimeError(
              f"Tokenizer generation failure. Vocab size of trained tokenizer is {tokenizer.get_vocab_size()}."
          )
  
      # Postprocess:
      cls_token_id = tokenizer.token_to_id("<cls>")
      sep_token_id = tokenizer.token_to_id("<sep>")
  
      # Generate template:
      type_id = str(1) if cfg_data.use_type_ids else str(0)
      single_template = "$A"
      if cfg_data.include_cls_token_in_corpus:
          single_template = "<cls> " + single_template
      if cfg_data.include_sep_token_in_corpus:
          single_template = single_template + " <sep>"
      tokenizer.post_processor = processors.TemplateProcessing(
          single=single_template,
          pair=f"<cls>:0 $A:0 <sep>:0 $B:{type_id} <sep>:{type_id}",
          special_tokens=[
              ("<cls>", cls_token_id),
              ("<sep>", sep_token_id),
          ],
      )
      # Wrap into fast codebase
      wrapped_tokenizer = PreTrainedTokenizerFast(
          tokenizer_object=tokenizer,
          model_max_length=cfg_data.seq_length,
          **special_token_args,
      )
      return wrapped_tokenizer
  
file_path: bert-ultrafast/training/cramming/data/tokenizer_preparation.py
imports: from transformers import AutoTokenizer, PreTrainedTokenizerFast
  from tokenizers import Tokenizer, models, normalizers, pre_tokenizers, decoders, trainers, Regex, processors
label_desc: Use image node_id 6 for calling a global function with example usage: _construct_tokenizer(raw_datasets, cfg_data, known_tokens) and returns: wrapped_tokenizer