code: def _move_stream_to_fixed_map(
      raw_data_streamed,
      max_entries_in_raw_dataset,
      max_raw_chunk_size=1e14,
  ):
      """Save streaming dataset to a fixed mapping-style database."""
      # I'm tired of IterableDatasets and will take the performance hit to write them out instead:
      try:
          if max_raw_chunk_size > max_entries_in_raw_dataset:
              with tempfile.TemporaryDirectory() as tmpdirname:
                  datasets.Dataset.from_dict(
                      dict(text=[v["text"] for v in raw_data_streamed])
                  ).save_to_disk(tmpdirname + "raw_data")
                  raw_data_mapped = datasets.load_from_disk(
                      tmpdirname + "raw_data"
                  )
              # This used to be only a move into RAM but this breaks memory later using C4:
              # raw_data = datasets.Dataset.from_dict(dict(text=[v["text"] for v in raw_data]))
              return raw_data_mapped
          else:
              with tempfile.TemporaryDirectory() as tmpdirname:
                  mapped_sets = []
                  data_in_RAM = defaultdict(list)
                  for idx, value_stream in enumerate(raw_data_streamed):
                      data_in_RAM["text"].append(value_stream["text"])
                      if ((idx + 1) % max_raw_chunk_size == 0) or (
                          (idx - 1) == max_entries_in_raw_dataset
                      ):
                          datasets.Dataset.from_dict(
                              data_in_RAM
                          ).save_to_disk(
                              tmpdirname + "raw_data" + str(idx)
                          )
                          mapped_dataset = datasets.load_from_disk(
                              tmpdirname + "raw_data" + str(idx)
                          )
                          log.info(
                              f"Saved temporary copy at idx {idx} of {max_entries_in_raw_dataset} at {tmpdirname + 'raw_data' + str(idx)}."
                          )
                          data_in_RAM["text"] = []
                          mapped_sets.append(mapped_dataset)
              return datasets.concatenate_datasets(mapped_sets)
      except OSError as e:
          detailed_OSError(e)
  
imports: import torch
  import datasets
  import hydra
  import os
  import contextlib
  import logging
  import tempfile
  from itertools import chain
  from collections import defaultdict
  import json
  from omegaconf import OmegaConf
  from .tokenizer_preparation import construct_tokenizer, load_tokenizer
  from .curriculum_sorting import _sort_tokenized_dataset_by_unigram, _sort_tokenized_dataset_by_token, _sort_tokenized_dataset_by_word_length
  from .deduplicate import deduplicate_huggingface_dataset
  from .utils import checksum_config, stage_dataset, detailed_OSError
label_desc: Use image node_id 3 for calling a global function with example usage: _move_stream_to_fixed_map(raw_data_streamed, max_entries_in_raw_dataset, max_raw_chunk_size) and returns: raw_data_mapped, datasets