code: def load_and_agg_data(
      src_root_dir: pathlib.Path,
      min_text_len: int,
      min_obs: int,
      seed: Optional[int],
      force: bool,
  ) -> list[tuple[str, str]]:
      """Download, load, and aggregate datasets."""
      iso_lang_resource = textacy.lang_id._datasets.IsoLangResource(
          src_root_dir.joinpath("iso-639")
      )
      iso_lang_resource.download(force=force)
      iso_lang_map = iso_lang_resource.load(
          exclude={"sh"}
      )  # TODO: why exclude sh?
      valid_langs = set(iso_lang_map.values())
  
      udhr = textacy.datasets.UDHR(src_root_dir.joinpath("udhr"))
      udhr.download(force=force)
      udhr_data = [
          (snippet, meta["lang"])
          for text, meta in udhr.records()
          for snippet in text.split("\n")
          if meta["lang"] in valid_langs
          and itertoolz.count(
              char for char in snippet if char.isalnum()
          )
          >= min_text_len
      ]
      random.shuffle(udhr_data)
  
      dslcc = textacy.lang_id._datasets.DSLCCDataset(
          src_root_dir.joinpath("dslcc")
      )
      dslcc.download(force=force)
      dslcc_data = dslcc.load(valid_langs, min_len=min_text_len)
  
      wili = textacy.lang_id._datasets.Wili2018Dataset(
          src_root_dir.joinpath("wili")
      )
      wili.download(force=force)
      wili_data = wili.load(iso_lang_map, min_len=min_text_len)
  
      tatoeba = textacy.lang_id._datasets.TatoebaDataset(
          src_root_dir.joinpath("tatoeba")
      )
      tatoeba.download(force=force)
      tatoeba_data = tatoeba.load(iso_lang_map, min_len=min_text_len)
  
      ted2020 = textacy.lang_id._datasets.Ted2020(
          src_root_dir.joinpath("ted2020")
      )
      ted2020.download(force=force)
      ted2020_data = ted2020.load(valid_langs, min_len=min_text_len)
  
      setimes = textacy.lang_id._datasets.SETimes(
          src_root_dir.joinpath("setimes")
      )
      setimes.download(force=force)
      setimes_data = setimes.load(valid_langs, min_len=min_text_len)
  
      ud = textacy.lang_id._datasets.UDDataset(
          src_root_dir.joinpath("ud")
      )
      ud.download(force=force)
      ud_data = ud.load(valid_langs, min_len=min_text_len)
  
      # aggregate and sample datasets
      agg_data = (
          udhr_data  # only has ~12k examples
          + get_random_sample(
              wili_data,
              len(wili_data),
              stratify=True,
              random_state=seed,
          )
          + get_random_sample(
              tatoeba_data, 2_500_000, stratify=True, random_state=seed
          )
          + get_random_sample(
              ted2020_data, 2_500_000, stratify=True, random_state=seed
          )
          + get_random_sample(
              ud_data, 2_500_000, stratify=True, random_state=seed
          )
          # add additional examples for hard-to-distinguish language groups
          + get_random_sample(
              dslcc_data, 100_000, stratify=True, random_state=seed
          )
          + get_random_sample(
              setimes_data, 200_000, stratify=True, random_state=seed
          )
      )
  
      agg_data = filter_data_by_lang_count(agg_data, min_obs)
  
      return agg_data
  
imports: import argparse
  import collections
  import logging
  import operator
  import pathlib
  import random
  import statistics
  from functools import partial
  from typing import Optional
  import sklearn.model_selection
  from toolz import itertoolz
  import textacy.datasets
  import textacy.io
  import textacy.lang_id._datasets
  import textacy.preprocessing
label_desc: Use image node_id 4 for calling a global function with example usage: load_and_agg_data(src_root_dir, min_text_len, min_obs, seed, force) and returns: agg_data