code: def main():
      args = add_and_parse_args()
      if args.save_dir:
          args.save_dir.mkdir(parents=True, exist_ok=True)
  
      data = load_and_agg_data(
          args.src_root_dir,
          args.min_text_len,
          args.min_obs,
          args.seed,
          args.force,
      )
      # HACK: let's make sure there aren't any URLs in our training data
      # since it seems like a bunch of characters that would confuse the model
      # let's also normalize the whitespace
      preproc = textacy.preprocessing.make_pipeline(
          partial(textacy.preprocessing.replace.urls, repl=""),
          textacy.preprocessing.normalize.whitespace,
          lambda x: x.replace("\n", " ").lower(),
      )
      data = ((preproc(text), lang) for text, lang in data)
      data = [
          item for item in data if len(item[0]) >= args.min_text_len
      ]
      summarize_data("agg", data)
  
      train_data, test_data = sklearn.model_selection.train_test_split(
          data,
          test_size=args.test_size,
          random_state=args.seed,
          stratify=[lang for _, lang in data],
      )
      test_data, valid_data = sklearn.model_selection.train_test_split(
          test_data,
          test_size=0.5,
          random_state=args.seed,
          stratify=[lang for _, lang in test_data],
      )
      print(
          f"training data: {len(train_data)}\n"
          f"test_data: {len(test_data)}\n"
          f"valid_data: {len(valid_data)}"
      )
  
      format_and_save_data(train_data, "train", args.save_dir)
      format_and_save_data(test_data, "test", args.save_dir)
      format_and_save_data(valid_data, "valid", args.save_dir)
  
imports: import argparse
  import collections
  import logging
  import operator
  import pathlib
  import random
  import statistics
  from functools import partial
  from typing import Optional
  import sklearn.model_selection
  from toolz import itertoolz
  import textacy.datasets
  import textacy.io
  import textacy.lang_id._datasets
  import textacy.preprocessing
label_desc: Use image node_id 1 for calling a global function with example usage: main() without return types