code: def main():
      parser = argparse.ArgumentParser(
          formatter_class=argparse.ArgumentDefaultsHelpFormatter,
      )
      parser.add_argument(
          "--root_dirpath",
          required=True,
          help="path to root directory under which datasets and models are saved",
      )
      parser.add_argument(
          "--twitter_creds",
          required=True,
          help="path to file containing twitter API credentials",
      )
      parser.add_argument(
          "--min_len",
          type=int,
          default=25,
          help="minimum number of alphanumeric characters in a text "
          "for it to be included in the training dataset",
      )
      parser.add_argument(
          "--min_obs",
          type=int,
          default=300,
          help="minimum number of observations -- (text, lang) pairs -- in a language "
          "for it to be included in the training dataset",
      )
      parser.add_argument(
          "--version",
          type=str,
          required=True,
          help="semantic version number to assign to trained model, e.g. '1.0'",
      )
      parser.add_argument(
          "--force",
          action="store_true",
          default=False,
          help="if specified, force downloads of all datasets, "
          "even if they already exist on disk under ``root_dirpath``",
      )
      args = parser.parse_args()
  
      root_dirpath = textacy.utils.to_path(args.root_dirpath).resolve()
      iso_639_dirpath = root_dirpath.joinpath("iso-639")
      twitter_dirpath = root_dirpath.joinpath("twitter")
      tatoeba_dirpath = root_dirpath.joinpath("tatoeba")
      wili_dirpath = root_dirpath.joinpath("wili")
      udhr_dirpath = root_dirpath.joinpath("udhr")
      dslcc_dirpath = root_dirpath.joinpath("dslcc")
      models_dirpath = root_dirpath.joinpath("models")
  
      # download, load, and summarize datasets
      download_iso_639_data(iso_639_dirpath, force=args.force)
      iso_639_data = load_iso_639_data(iso_639_dirpath, exclude=("sh",))
      print(sorted(iso_639_data.items())[:5], "...")
  
      download_twitter_data(
          twitter_dirpath, args.twitter_creds, force=args.force
      )
      twitter_data = load_twitter_data(
          twitter_dirpath,
          set(iso_639_data.values()),
          min_len=args.min_len,
      )
      summarize_dataset(twitter_data)
  
      download_tatoeba_data(tatoeba_dirpath, force=args.force)
      tatoeba_data = load_tatoeba_data(
          tatoeba_dirpath, iso_639_data, min_len=args.min_len
      )
      summarize_dataset(tatoeba_data)
  
      download_wili_data(wili_dirpath, force=args.force)
      wili_data = load_wili_data(
          wili_dirpath, iso_639_data, min_len=args.min_len
      )
      summarize_dataset(wili_data)
  
      download_udhr_data(udhr_dirpath, force=args.force)
      udhr_data = load_udhr_data(
          udhr_dirpath, set(iso_639_data.values()), min_len=args.min_len
      )
      summarize_dataset(udhr_data)
  
      download_dslcc_data(dslcc_dirpath, force=args.force)
      dslcc_data = load_dslcc_data(
          dslcc_dirpath,
          set(iso_639_data.values()),
          min_len=args.min_len,
      )
      summarize_dataset(dslcc_data)
  
      # HACK HACK HACK
      leipzig_dirpath = textacy.utils.to_path(
          "/Users/burtondewilde/Desktop/datasets/language_identification/leipzig-corpora"
      ).resolve()
      if leipzig_dirpath.is_dir():
          leipzig_data = load_leipzig_data(
              leipzig_dirpath, iso_639_data, min_len=args.min_len
          )
          summarize_dataset(leipzig_data)
      else:
          logging.warning("leipzig data hack unavailable, sorry")
          leipzig_data = []
  
      # aggregate and sample datasets
      datasets = (
          udhr_data
          + wili_data
          + get_random_sample(
              tatoeba_data, 420000, stratify=True, random_state=42
          )
          + get_random_sample(
              leipzig_data, 480000, stratify=True, random_state=42
          )
          + get_random_sample(
              twitter_data,
              len(twitter_data),
              stratify=True,
              random_state=42,
          )
          + get_random_sample(
              dslcc_data, 20000, stratify=True, random_state=42
          )
          + get_random_sample(
              [item for item in tatoeba_data if item[1] == "en"],
              10000,
              stratify=False,
              random_state=42,
          )
      )
      datasets = get_random_sample(datasets, 3000000, stratify=True)
  
      valid_langs = {
          lang
          for lang, count in collections.Counter(
              lang for _, lang in datasets
          ).most_common()
          if count >= args.min_obs
      }
      datasets = [
          text_lang
          for text_lang in datasets
          if text_lang[1] in valid_langs
      ]
  
      chars_set = {char for text, _ in datasets for char in text}
      langs_set = {lang for _, lang in datasets}
      num_langs = len(langs_set)
      print("# unique chars:", len(chars_set))
      print("# unique langs:", num_langs)
      summarize_dataset(datasets)
  
      (
          train_items,
          test_items,
      ) = sklearn.model_selection.train_test_split(
          datasets, test_size=0.2, random_state=42
      )
      print(train_items[0])
      print("# train items:", len(train_items))
      print("# test items:", len(test_items))
  
      # fit and validate a model
      model_id = "lang-identifier-v{}-sklearn-v{}".format(
          args.version, sklearn.__version__[:4]
      )
      print("training model {} ...".format(model_id))
      lid = textacy.lang_utils.LangIdentifier(data_dir=models_dirpath)
      lid.init_pipeline()
      print(lid.pipeline.steps)
      lid.pipeline.fit(
          [text for text, _ in train_items],
          y=[lang for _, lang in train_items],
      )
      true, preds = test_model(
          lid.pipeline,
          test_items,
          filepath=models_dirpath.joinpath(model_id + ".txt"),
      )
      try:
          ax = plot_confusion_matrix(
              true,
              preds,
              normalize=True,
              title=None,
              cmap=plt.cm.Blues,
              annotate=False,
          )
          ax.get_figure().savefig(
              models_dirpath.joinpath(
                  model_id + "-confusion-matrix.png"
              )
          )
      except Exception:
          pass  # well, at least we tried
      joblib.dump(
          lid.pipeline,
          models_dirpath.joinpath(model_id + ".pkl.gz"),
          compress=3,
      )
  
imports: import argparse
  import collections
  import logging
  import operator
  import os
  import random
  import re
  import statistics
  import sys
  import tarfile
  from pprint import pprint
  import joblib
  import matplotlib.pyplot
  import numpy
  import sklearn
  import sklearn.metrics
  import sklearn.model_selection
  import tqdm
  import twitter
  import yaml
  from cytoolz import itertoolz
  import textacy
  import textacy.datasets
  import textacy.io
  import textacy.preprocessing
label_desc: Use image node_id 1 for calling a global function with example usage: main() without return types