code: def load_and_agg_data(
      root_dirpath: pathlib.Path,
      force: bool,
      min_len: int,
      min_obs: int,
  ) -> List[Tuple[str, str]]:
      """Download, load, and aggregate datasets."""
      iso_lang_resource = textacy.lang_id_._datasets.IsoLangResource(
          root_dirpath.joinpath("iso-639")
      )
      iso_lang_resource.download(force=force)
      iso_lang_map = iso_lang_resource.load(
          exclude={"sh"}
      )  # TODO: why exclude sh?
      valid_langs = set(iso_lang_map.values())
  
      udhr = textacy.datasets.UDHR(root_dirpath.joinpath("udhr"))
      udhr.download(force=force)
      udhr_data = [
          (snippet, meta["lang"])
          for text, meta in udhr.records()
          for snippet in text.split("\n")
          if meta["lang"] in valid_langs
          and itertoolz.count(
              char for char in snippet if char.isalnum()
          )
          >= min_len
      ]
  
      dslcc = textacy.lang_id_._datasets.DSLCCDataset(
          root_dirpath.joinpath("dslcc")
      )
      dslcc.download(force=force)
      dslcc_data = dslcc.load(valid_langs, min_len=min_len)
  
      wili = textacy.lang_id_._datasets.Wili2018Dataset(
          root_dirpath.joinpath("wili")
      )
      wili.download(force=force)
      wili_data = wili.load(iso_lang_map, min_len=min_len)
  
      tatoeba = textacy.lang_id_._datasets.TatoebaDataset(
          root_dirpath.joinpath("tatoeba")
      )
      tatoeba.download(force=force)
      tatoeba_data = tatoeba.load(iso_lang_map, min_len=min_len)
  
      ud = textacy.lang_id_._datasets.UDDataset(
          root_dirpath.joinpath("ud")
      )
      ud.download(force=force)
      ud_data = ud.load(valid_langs, min_len=min_len)
  
      # aggregate and sample datasets
      agg_data = (
          udhr_data
          + wili_data
          + get_random_sample(
              tatoeba_data, 200000, stratify=True, random_state=42
          )
          + get_random_sample(
              ud_data, 200000, stratify=True, random_state=42
          )
          # add additional examples for hard-to-distinguish language groups
          + get_random_sample(
              dslcc_data, 50000, stratify=True, random_state=42
          )
          # add some extra english examples, since there's apparently a fair amount
          # of english sprinkled throughout other languages, causing meh performance
          + get_random_sample(
              [item for item in tatoeba_data if item[1] == "en"],
              10000,
              stratify=False,
              random_state=42,
          )
      )
      agg_data = filter_data_by_lang_count(agg_data, min_obs)
      return agg_data
  
imports: from __future__ import annotations
  import argparse
  import collections
  import logging
  import operator
  import pathlib
  import random
  import statistics
  from typing import List, Tuple
  import sklearn.metrics
  import sklearn.model_selection
  import sklearn.preprocessing
  import thinc.api
  from cytoolz import itertoolz
  import textacy
  import textacy.datasets
  import textacy.lang_id_
  import textacy.lang_id_._datasets
  import textacy.lang_id_._training
  import textacy.preprocessing
label_desc: Use image node_id 3 for calling a global function with example usage: load_and_agg_data(root_dirpath, force, min_len, min_obs) and returns: agg_data