code: def _huggingface_preprocessing(
      raw_dataset, tokenizer, cfg_data, num_threads=4
  ):
      """Dataset preprocessing and tokenization.
  
      This is basically the default HF routine from
      https://github.com/huggingface/transformers/blob/master/examples/pytorch/language-modeling/run_mlm.py
      """
      # Preprocessing the datasets.
      # First we tokenize all the texts.
      column_names = getattr(raw_dataset, "column_names", "text")
      text_column_name = (
          "text" if "text" in column_names else column_names[0]
      )
  
      max_seq_length = tokenizer.model_max_length
      map_setup = dict(
          batched=True,
          batch_size=1024,
          num_proc=num_threads if num_threads > 0 else None,
          # load_from_cache_file=False,
          # keep_in_memory=False,
      )
      parellism_flag = os.environ["TOKENIZERS_PARALLELISM"]
      if num_threads > 0:
          os.environ["TOKENIZERS_PARALLELISM"] = "false"
      # Otherwise, we tokenize every text, then concatenate them together before splitting them in smaller parts.
      # The Collator is modified not to read special_masks anyway:
  
      def tokenize_function(examples):
          return tokenizer(
              examples[text_column_name],
              return_special_tokens_mask=False,
              return_attention_mask=False,
              return_token_type_ids=cfg_data.use_type_ids,
          )
  
      tokenizer.model_max_length = 1e30
      tokenized_dataset = raw_dataset.map(
          tokenize_function,
          remove_columns=column_names,
          desc="Running tokenizer on every text in dataset",
          **map_setup,
      )
      tokenizer.model_max_length = max_seq_length
  
      # Main data processing function that will concatenate all texts from our dataset and generate chunks of
      # max_seq_length.
      def group_texts(examples):
          # Concatenate all texts.
          concatenated_examples = {
              k: list(chain(*examples[k])) for k in examples.keys()
          }
          total_length = len(
              concatenated_examples[list(examples.keys())[0]]
          )
          # We drop the small remainder, we could add padding if the model supported it instead of this drop, you can
          # customize this part to your needs.
          if total_length >= max_seq_length:
              total_length = (
                  total_length // max_seq_length
              ) * max_seq_length
          # Split by chunks of max_len.
          result = {
              k: [
                  t[i : i + max_seq_length]
                  for i in range(0, total_length, max_seq_length)
              ]
              for k, t in concatenated_examples.items()
          }
          return result
  
      tokenized_dataset = tokenized_dataset.map(
          group_texts,
          desc=f"Grouping texts in chunks of {max_seq_length}",
          **map_setup,
      )
  
      # Shuffle?
      if cfg_data.ordering == "randomized":
          tokenized_dataset = tokenized_dataset.shuffle(seed=233)
      elif cfg_data.ordering == "unigram-curriculum":
          tokenized_dataset = _sort_tokenized_dataset_by_unigram(
              tokenized_dataset, tokenizer, num_threads
          )
      elif cfg_data.ordering == "reverse-unigram-curriculum":
          tokenized_dataset = _sort_tokenized_dataset_by_unigram(
              tokenized_dataset, tokenizer, num_threads, reverse=True
          )
      elif cfg_data.ordering == "word-length-curriculum":
          tokenized_dataset = _sort_tokenized_dataset_by_word_length(
              tokenized_dataset, tokenizer, num_threads
          )
      elif cfg_data.ordering == "sentence-length-curriculum":
          tokenized_dataset = _sort_tokenized_dataset_by_token(
              tokenized_dataset,
              tokenizer,
              tokenizer.encode(".")[0],
              num_threads,
          )
      elif cfg_data.ordering == "fragment-curriculum":
          tokenized_dataset = _sort_tokenized_dataset_by_token(
              tokenized_dataset,
              tokenizer,
              tokenizer.encode("<sep>")[0],
              num_threads,
          )
      else:
          raise ValueError(
              f"Invalid dataset ordering {cfg_data.ordering} provided."
          )
  
      # Reduce size to maximal limit:
      if cfg_data.max_seq_in_tokenized_dataset < len(tokenized_dataset):
          tokenized_dataset = tokenized_dataset.select(
              range(int(cfg_data.max_seq_in_tokenized_dataset)),
              keep_in_memory=True,
          )
  
      # Finally flatten
      # This is necessary for the save_to_disk call that comes next. If skipped here, the call will be invoked from save_to_disk
      # This way, atleast it shares the same batch parameters and prints a progress bar.
      tokenized_dataset = tokenized_dataset.map(
          desc="Flattening the indices", **map_setup
      )
      os.environ["TOKENIZERS_PARALLELISM"] = parellism_flag
      return tokenized_dataset
  
imports: import torch
  import datasets
  import hydra
  import os
  import contextlib
  import logging
  import tempfile
  from itertools import chain
  from collections import defaultdict
  import json
  from omegaconf import OmegaConf
  from .tokenizer_preparation import construct_tokenizer, load_tokenizer
  from .curriculum_sorting import _sort_tokenized_dataset_by_unigram, _sort_tokenized_dataset_by_token, _sort_tokenized_dataset_by_word_length
  from .deduplicate import deduplicate_huggingface_dataset
  from .utils import checksum_config, stage_dataset, detailed_OSError
label_desc: Use image node_id 4 for calling a global function with example usage: _huggingface_preprocessing(raw_dataset, tokenizer, cfg_data, num_threads) and returns: tokenized_dataset, tokenizer, result