code: def main(args):
      # Download the data
      url = _URLS[args.dataset]
      file_hash = _URL_FILE_STATS[url]
      target_download_location = os.path.join(
          args.cache_path, os.path.basename(url)
      )
      download(url, target_download_location, sha1_hash=file_hash)
      save_dir = (
          args.dataset if args.save_dir is None else args.save_dir
      )
      if not os.path.exists(save_dir):
          os.makedirs(save_dir, exist_ok=True)
      # Extract and process the data
      if args.dataset == "wikitext2":
          with zipfile.ZipFile(target_download_location) as zf:
              train_data = zf.read("wikitext-2/wiki.train.tokens")
              valid_data = zf.read("wikitext-2/wiki.valid.tokens")
              test_data = zf.read("wikitext-2/wiki.test.tokens")
              for filename, part in [
                  ("train.txt", train_data),
                  ("valid.txt", valid_data),
                  ("test.txt", test_data),
              ]:
                  filename = os.path.join(save_dir, filename)
                  print(
                      "{} will have {} bytes".format(
                          filename, len(part)
                      )
                  )
                  if not path_exist_and_skip(filename, args.overwrite):
                      with open(filename, "wb") as of:
                          of.write(part)
              vocab = build_vocab(
                  [
                      os.path.join(save_dir, "train.txt"),
                      os.path.join(save_dir, "valid.txt"),
                      os.path.join(save_dir, "test.txt"),
                  ]
              )
              vocab.save(os.path.join(save_dir, "vocab.json"))
      elif args.dataset == "wikitext103":
          with zipfile.ZipFile(target_download_location) as zf:
              train_data = zf.read("wikitext-103/wiki.train.tokens")
              valid_data = zf.read("wikitext-103/wiki.valid.tokens")
              test_data = zf.read("wikitext-103/wiki.test.tokens")
              for filename, part in [
                  ("train.txt", train_data),
                  ("valid.txt", valid_data),
                  ("test.txt", test_data),
              ]:
                  filename = os.path.join(save_dir, filename)
                  if not path_exist_and_skip(filename, args.overwrite):
                      print(
                          "{} will have {} bytes".format(
                              filename, len(part)
                          )
                      )
                      with open(filename, "wb") as of:
                          of.write(part)
              vocab = build_vocab([os.path.join(save_dir, "train.txt")])
              vocab.save(os.path.join(save_dir, "vocab.json"))
      elif args.dataset == "text8":
          with zipfile.ZipFile(target_download_location) as zf:
              with zf.open("text8", "r") as f:
                  data = f.read().decode("utf-8")
                  num_test_chars = 5000000
                  train_data = data[: -2 * num_test_chars]
                  valid_data = data[
                      -2 * num_test_chars : -num_test_chars
                  ]
                  test_data = data[-num_test_chars:]
                  for filename, part in [
                      ("train.txt", train_data),
                      ("valid.txt", valid_data),
                      ("test.txt", test_data),
                  ]:
                      filename = os.path.join(save_dir, filename)
                      print(
                          "{} will have {} bytes".format(
                              filename, len(part)
                          )
                      )
                      print("- Tokenizing...")
                      # Change space ' ' to underscore '_'
                      part_str = " ".join(
                          ["_" if c == " " else c for c in part.strip()]
                      )
                      print("- Writing...")
                      if not path_exist_and_skip(
                          filename, args.overwrite
                      ):
                          with open(
                              filename, "w", encoding="utf-8"
                          ) as of:
                              of.write(part_str)
                      if not path_exist_and_skip(
                          filename + ".raw", args.overwrite
                      ):
                          with open(
                              filename + ".raw", "w", encoding="utf-8"
                          ) as of:
                              of.write(part)
              vocab = build_vocab(
                  [os.path.join(save_dir, "train.txt")], eos_token=None
              )
              vocab.save(os.path.join(save_dir, "vocab.json"))
      elif args.dataset == "enwik8":
          with zipfile.ZipFile(target_download_location) as zf:
              data = zf.read("enwik8")
              print("Length of enwik8: {}".format(len(data)))
              num_test_chars = 5000000
              train_data = data[: -2 * num_test_chars]
              valid_data = data[-2 * num_test_chars : -num_test_chars]
              test_data = data[-num_test_chars:]
  
              for filename, part in [
                  ("train.txt", train_data),
                  ("valid.txt", valid_data),
                  ("test.txt", test_data),
              ]:
                  filename = os.path.join(save_dir, filename)
                  print(
                      "{} will have {} bytes".format(
                          filename, len(part)
                      )
                  )
                  print("- Tokenizing...")
                  part_str = " ".join(
                      [str(c) if c != ord("\n") else "\n" for c in part]
                  )
                  print("- Writing...")
                  if not path_exist_and_skip(filename, args.overwrite):
                      with open(filename, "w") as of:
                          of.write(part_str)
                  if not path_exist_and_skip(
                      filename + ".raw", args.overwrite
                  ):
                      with open(filename + ".raw", "wb") as of:
                          of.write(part)
              vocab = build_vocab(
                  [os.path.join(save_dir, "train.txt")], eos_token=None
              )
              vocab.save(os.path.join(save_dir, "vocab.json"))
  
      elif args.dataset == "gbw":
          vocab_path = download(
              _URLS["gbw_vocab"],
              os.path.join(args.cache_path, "1b_word_vocab.txt"),
              sha1_hash=_URL_FILE_STATS[_URLS["gbw_vocab"]],
          )
          with tarfile.open(target_download_location) as f:
              os.makedirs(
                  os.path.join(save_dir, "train"), exist_ok=True
              )
              os.makedirs(os.path.join(save_dir, "test"), exist_ok=True)
              for member in f.getmembers():
                  if (
                      "training-monolingual.tokenized.shuffled"
                      in member.name
                      and "news.en" in member.name
                  ):
                      basename = os.path.basename(member.name)
                      with f.extractfile(member) as f_in:
                          with open(
                              os.path.join(save_dir, "train", basename),
                              "wb",
                          ) as f_out:
                              shutil.copyfileobj(f_in, f_out)
                  elif (
                      "heldout-monolingual.tokenized.shuffled"
                      in member.name
                      and ".heldout-" in member.name
                  ):
                      basename = os.path.basename(member.name)
                      with f.extractfile(member) as f_in:
                          with open(
                              os.path.join(save_dir, "test", basename),
                              "wb",
                          ) as f_out:
                              shutil.copyfileobj(f_in, f_out)
          all_tokens = []
          with open(vocab_path, "r") as f:
              for token in f:
                  token = token.strip().split()[0]
                  all_tokens.append(token)
          vocab = Vocab(all_tokens, bos_token="<S>", unk_token="<UNK>")
          vocab.save(os.path.join(save_dir, "vocab.json"))
          print("Saved Google-One-Billion-Word in {}".format(save_dir))
          print("Vocab={}".format(vocab))
      else:
          raise NotImplementedError
  
file_path: gluon-nlp/scripts/datasets/language_modeling/prepare_lm.py
imports: import argparse
  import os
  import zipfile
  import tarfile
  import shutil
  from typing import List, Optional
  from collections import Counter
  from gluonnlp.base import get_data_home_dir
  from gluonnlp.utils.misc import download, load_checksum_stats
  from gluonnlp.data.vocab import Vocab
label_desc: Use image node_id 4 for calling a global function with example usage: main(args) without return types