class_docstr: Dataset based on paragraphs from Wikipedia in 230+ languages.
  
  Source: https://zenodo.org/record/841984
  
  References:
      Thoma, Martin. "The WiLI benchmark dataset for written language identification."
      arXiv preprint arXiv:1801.07779 (2018).
class_docstr_tok: ["Dataset","based","on","paragraphs","from","Wikipedia","in","230+","languages",".","Source",":","https",":","\/\/zenodo.org\/record\/841984","References",":","Thoma",",","Martin",".","``","The","WiLI","benchmark","dataset","for","written","language","identification",".","''","arXiv","preprint","arXiv:1801.07779","(","2018",")","."]
docstr: Args:
      iso_lang_map
      min_len: Minimum text length in *chars* for a given example to be included.
  
  Returns:
      Sequence of (text, lang) examples.
docstr_tok: ["Args",":","iso_lang_map","min_len",":","Minimum","text","length","in","*","chars","*","for","a","given","example","to","be","included",".","Returns",":","Sequence","of","(","text",",","lang",")","examples","."]
code: def load(
      self,
      iso_lang_map: Dict[str, str],
      min_len: int = 25,
  ) -> List[Tuple[str, str]]:
      """
      Args:
          iso_lang_map
          min_len: Minimum text length in *chars* for a given example to be included.
  
      Returns:
          Sequence of (text, lang) examples.
      """
      data: list[tuple[str, str]] = []
      # we'll combine train/test from individual datasets
      # and instead split on the full, aggregated dataset
      for subset in ("train", "test"):
          text_lines = tio.read_text(
              self.data_dir.joinpath(f"x_{subset}.txt"), lines=True
          )
          lang_lines = tio.read_text(
              self.data_dir.joinpath(f"y_{subset}.txt"), lines=True
          )
          texts = (line.strip() for line in text_lines)
          langs = (line.strip() for line in lang_lines)
          data.extend(
              (text, iso_lang_map[lang])
              for text, lang in zip(texts, langs)
              if lang in iso_lang_map
              and itertoolz.count(
                  char for char in text if char.isalnum()
              )
              >= min_len
          )
      LOGGER.info("loaded Wili2018Dataset data:\n%s ...", data[:3])
      return data
  
imports: from __future__ import annotations
  import logging
  import operator
  import pathlib
  import random
  import re
  from typing import Dict, Iterable, List, Optional, Set, Tuple
  from cytoolz import itertoolz
  import textacy
  import textacy.utils
  from textacy import io
label_desc: Use image node_id 3 for calling the Wili2018Dataset obj's underlying member method code with example usage: obj.load(iso_lang_map, min_len) and returns: data