code: def download_twitter_data(dirpath, creds_fpath, force=False):
      """
      Download two collections of (lang, tweet_id) pairs from Twitter --
      a "uniformly sampled" collection of ~120k tweets over all languages and
      a "recall oriented" collection of ~1.5k tweets per language --
      then fetch available tweets' data from the Twitter API.
  
      Args:
          dirpath (str or :class:`pathlib.Path`)
          creds_fpath (str or :class:`pathlib.Path`)
          force (bool)
  
      References:
          https://blog.twitter.com/engineering/en_us/a/2015/evaluating-language-identification-performance.html
  
      TODO: Ideally, use a tweet search endpoint and filter by language,
      then just iterate over all ISO-639-1 language codes.
      """
      dirpath = textacy.utils.to_path(dirpath).resolve()
      url_fnames = [
          (
              "https://raw.githubusercontent.com/mitjat/langid_eval/master/uniformly_sampled.tsv",
              "uniformly_sampled.tsv",
          ),
          (
              "https://raw.githubusercontent.com/mitjat/langid_eval/master/recall_oriented.tsv",
              "recall_oriented.tsv",
          ),
      ]
      # download tweet ids first
      for url, fname in url_fnames:
          textacy.io.download_file(
              url, filename=fname, dirpath=dirpath, force=force
          )
      # download full tweets data next
      tweets_fpath = dirpath.joinpath("tweets.jsonl")
      if tweets_fpath.is_file() and force is False:
          logging.info(
              "tweets data already downloaded to %s", tweets_fpath
          )
          return
  
      # load twitter ids data from disk
      tweet_lang_ids = []
      for fname in ["uniformly_sampled.tsv", "recall_oriented.tsv"]:
          tweet_lang_ids.extend(
              textacy.io.read_csv(
                  dirpath.joinpath(fname),
                  delimiter="\t",
                  fieldnames=["lang", "status_id"],
                  quoting=1,
              )
          )
      logging.info("loaded %s tweet ids from disk", len(tweet_lang_ids))
      # parse status ids
      status_ids = set()
      for row in tweet_lang_ids:
          try:
              status_ids.add(int(row["status_id"]))
          # there are a small handful of bad status ids, shrug
          except ValueError:
              pass
      logging.info(
          "... of which %s had valid, unique ids", len(status_ids)
      )
      status_ids = list(status_ids)
      # instantiate twitter api client
      with textacy.utils.to_path(creds_fpath).resolve().open(
          mode="rt"
      ) as f:
          creds = yaml.safe_load(f.read())
      api = twitter.Api(sleep_on_rate_limit=True, **creds)
      # get tweets data in chunks
      chunk_size = 100
      pbar = tqdm.tqdm(total=len(status_ids), unit="tweets")
      tweets = []
      try:
          for chunk_ids in itertoolz.partition_all(
              chunk_size, status_ids
          ):
              chunk_tweets = api.GetStatuses(
                  chunk_ids,
                  trim_user=True,
                  include_entities=True,
                  map=False,
              )
              tweets.extend(chunk_tweets)
              pbar.update(len(chunk_ids))
      except Exception:
          logging.exception(
              "encountered an error while downloading tweets"
          )
      finally:
          pbar.close()
          tweets = [tweet.AsDict() for tweet in tweets]
          logging.info("downloaded data for %s tweets", len(tweets))
          textacy.io.write_json(
              tweets, tweets_fpath, mode="wt", lines=True
          )
  
imports: import argparse
  import collections
  import logging
  import operator
  import os
  import random
  import re
  import statistics
  import sys
  import tarfile
  from pprint import pprint
  import joblib
  import matplotlib.pyplot
  import numpy
  import sklearn
  import sklearn.metrics
  import sklearn.model_selection
  import tqdm
  import twitter
  import yaml
  from cytoolz import itertoolz
  import textacy
  import textacy.datasets
  import textacy.io
  import textacy.preprocessing
label_desc: Use image node_id 4 for calling a global function with example usage: download_twitter_data(dirpath, creds_fpath, force) without return types