code: def main():
      args = add_and_parse_args()
  
      lang_identifier = textacy.lang_id_.LangIdentifier(
          textacy.lang_id_.models.LangIdentifierModelV2(),
          args.root_dirpath.joinpath("lang_identifier"),
          version=args.version,
      )
  
      data = load_and_agg_data(
          args.root_dirpath, args.force, args.min_len, args.min_obs
      )
      # HACK: let's make sure there aren't any URLs in our training data
      # since it seems like a bunch of characters that would confuse the model
      data = [
          (textacy.preprocessing.replace.urls(text, ""), lang)
          for text, lang in data
      ]
      summarize_data("agg", data)
  
      train_data, test_data = sklearn.model_selection.train_test_split(
          data,
          test_size=0.2,
          random_state=42,
          stratify=[lang for _, lang in data],
      )
      print(
          f"training data: {len(train_data)}\ntest_data: {len(test_data)}"
      )
  
      batch_size = thinc.api.compounding(2.0, 64.0, 1.001)
      # NOTE: training appears to be VERY sensitive to learning rate
      # these values have been manually fine-tuned ...
      # learn_rate = thinc.api.cyclic_triangular(min_lr=0.0005, max_lr=0.005, period=2500)
      learn_rate = (
          textacy.lang_id_._training.decaying_cyclic_triangular(
              thinc.api.decaying(1.0, decay=5e-5),
              thinc.api.cyclic_triangular(
                  min_lr=0.0005, max_lr=0.0025, period=3000
              ),
              min_lr=0.00025,
          )
      )
      model = textacy.lang_id_._training.train_model(
          lang_identifier._model_base,
          train=train_data,
          test=test_data,
          n_iter=args.n_iter,
          batch_size=batch_size,
          learn_rate=learn_rate,
      )
      lang_identifier._model = model
      if args.save:
          lang_identifier.save_model()
  
imports: from __future__ import annotations
  import argparse
  import collections
  import logging
  import operator
  import pathlib
  import random
  import statistics
  from typing import List, Tuple
  import sklearn.metrics
  import sklearn.model_selection
  import sklearn.preprocessing
  import thinc.api
  from cytoolz import itertoolz
  import textacy
  import textacy.datasets
  import textacy.lang_id_
  import textacy.lang_id_._datasets
  import textacy.lang_id_._training
  import textacy.preprocessing
label_desc: Use image node_id 1 for calling a global function with example usage: main() without return types