code: def main(cfg) -> None:
      logging.info(
          "\n\n************** Experiment configuration ***********"
      )
      logging.info(OmegaConf.to_yaml(cfg, resolve=True))
  
      if cfg.output_file is None:
          cfg.output_file = "huggingface_text_corpus.txt"
  
      if Path(cfg.output_file).exists():
          logging.info(
              f"Output file {cfg.output_file} already exists, removing it..."
          )
          os.system(f"rm {cfg.output_file}")
  
      for data_cfg in cfg.hf_data_cfg:
          if "num_proc" in data_cfg and data_cfg.get(
              "streaming", False
          ):
              logging.warning(
                  "num_proc is not supported for streaming datasets, removing it from config"
              )
              with open_dict(data_cfg):
                  data_cfg.pop("num_proc")
          logging.info(
              f"Loading from HuggingFace datasets library with config: {OmegaConf.to_container(data_cfg, resolve=True)}"
          )
          dataset = hf_datasets.load_dataset(**data_cfg)
          logging.info("Start extracting text from dataset...")
          with Pool(cfg.num_workers) as p:
              text_corpus = p.map(worker, zip(dataset, repeat(cfg)))
              with Path(cfg.output_file).open("a") as f:
                  for line in text_corpus:
                      f.write(f"{line}\n")
              logging.info(
                  f"Finished processing {len(text_corpus)} samples from {data_cfg}"
              )
      logging.info("All Done!")
  
file_path: NeMo/scripts/tokenizers/get_hf_text_data.py
imports: import os
  from itertools import repeat
  from multiprocessing import Pool
  from pathlib import Path
  import datasets
  from omegaconf import OmegaConf, open_dict
  from nemo.core.config import hydra_runner
  from nemo.utils import logging
label_desc: Use image node_id 4 for calling a global function with example usage: main(cfg) without return types