class_docstr: Treat a Wikipedia articles dump as a read-only, streamed, memory-efficient corpus.
  
  Supported dump formats:
  
  * <LANG>wiki-<YYYYMMDD>-pages-articles.xml.bz2
  * <LANG>wiki-latest-pages-articles.xml.bz2
  
  The documents are extracted on-the-fly, so that the whole (massive) dump can stay compressed on disk.
  
  Notes
  -----
  Dumps for the English Wikipedia can be founded at https://dumps.wikimedia.org/enwiki/.
  
  Attributes
  ----------
  metadata : bool
      Whether to write articles titles to serialized corpus.
  
  Warnings
  --------
  "Multistream" archives are *not* supported in Python 2 due to `limitations in the core bz2 library
  <https://docs.python.org/2/library/bz2.html#de-compression-of-files>`_.
  
  Examples
  --------
  .. sourcecode:: pycon
  
      >>> from gensim.test.utils import datapath, get_tmpfile
      >>> from gensim.corpora import WikiCorpus, MmCorpus
      >>>
      >>> path_to_wiki_dump = datapath("enwiki-latest-pages-articles1.xml-p000000010p000030302-shortened.bz2")
      >>> corpus_path = get_tmpfile("wiki-corpus.mm")
      >>>
      >>> wiki = WikiCorpus(path_to_wiki_dump)  # create word->word_id mapping, ~8h on full wiki
      >>> MmCorpus.serialize(corpus_path, wiki)  # another 8h, creates a file in MatrixMarket format and mapping
class_docstr_tok: ["Treat","a","Wikipedia","articles","dump","as","a","read-only",",","streamed",",","memory-efficient","corpus",".","Supported","dump","formats",":","*","<","LANG",">","wiki-","<","YYYYMMDD",">","-pages-articles.xml.bz2","*","<","LANG",">","wiki-latest-pages-articles.xml.bz2","The","documents","are","extracted","on-the-fly",",","so","that","the","whole","(","massive",")","dump","can","stay","compressed","on","disk",".","Notes","--","--","-","Dumps","for","the","English","Wikipedia","can","be","founded","at","https",":","\/\/dumps.wikimedia.org\/enwiki\/",".","Attributes","--","--","--","--","--","metadata",":","bool","Whether","to","write","articles","titles","to","serialized","corpus",".","Warnings","--","--","--","--","''","Multistream","''","archives","are","*","not","*","supported","in","Python","2","due","to","`","limitations","in","the","core","bz2","library","<","https",":","\/\/docs.python.org\/2\/library\/bz2.html","#","de-compression-of-files",">","`","_",".","Examples","--","--","--","--","..","sourcecode",":",":","pycon",">",">",">","from","gensim.test.utils","import","datapath",",","get_tmpfile",">",">",">","from","gensim.corpora","import","WikiCorpus",",","MmCorpus",">",">",">",">",">",">","path_to_wiki_dump","=","datapath","(","``","enwiki-latest-pages-articles1.xml-p000000010p000030302-shortened.bz2","''",")",">",">",">","corpus_path","=","get_tmpfile","(","``","wiki-corpus.mm","''",")",">",">",">",">",">",">","wiki","=","WikiCorpus","(","path_to_wiki_dump",")","#","create","word-",">","word_id","mapping",",","~8h","on","full","wiki",">",">",">","MmCorpus.serialize","(","corpus_path",",","wiki",")","#","another","8h",",","creates","a","file","in","MatrixMarket","format","and","mapping"]
docstr: Iterate over the dump, yielding a list of tokens for each article that passed
  the length and namespace filtering.
  
  Uses multiprocessing internally to parallelize the work and process the dump more quickly.
  
  Notes
  -----
  This iterates over the **texts**. If you want vectors, just use the standard corpus interface
  instead of this method:
  
  Examples
  --------
  .. sourcecode:: pycon
  
      >>> from gensim.test.utils import datapath
      >>> from gensim.corpora import WikiCorpus
      >>>
      >>> path_to_wiki_dump = datapath("enwiki-latest-pages-articles1.xml-p000000010p000030302-shortened.bz2")
      >>>
      >>> for vec in WikiCorpus(path_to_wiki_dump):
      ...     pass
  
  Yields
  ------
  list of str
      If `metadata` is False, yield only list of token extracted from the article.
  (list of str, (int, str))
      List of tokens (extracted from the article), page id and article title otherwise.
docstr_tok: ["Iterate","over","the","dump",",","yielding","a","list","of","tokens","for","each","article","that","passed","the","length","and","namespace","filtering",".","Uses","multiprocessing","internally","to","parallelize","the","work","and","process","the","dump","more","quickly",".","Notes","--","--","-","This","iterates","over","the","*","*","texts","*","*",".","If","you","want","vectors",",","just","use","the","standard","corpus","interface","instead","of","this","method",":","Examples","--","--","--","--","..","sourcecode",":",":","pycon",">",">",">","from","gensim.test.utils","import","datapath",">",">",">","from","gensim.corpora","import","WikiCorpus",">",">",">",">",">",">","path_to_wiki_dump","=","datapath","(","``","enwiki-latest-pages-articles1.xml-p000000010p000030302-shortened.bz2","''",")",">",">",">",">",">",">","for","vec","in","WikiCorpus","(","path_to_wiki_dump",")",":","...","pass","Yields","--","--","--","list","of","str","If","`","metadata","`","is","False",",","yield","only","list","of","token","extracted","from","the","article",".","(","list","of","str",",","(","int",",","str",")",")","List","of","tokens","(","extracted","from","the","article",")",",","page","id","and","article","title","otherwise","."]
code: def get_texts(self):
      """Iterate over the dump, yielding a list of tokens for each article that passed
      the length and namespace filtering.
  
      Uses multiprocessing internally to parallelize the work and process the dump more quickly.
  
      Notes
      -----
      This iterates over the **texts**. If you want vectors, just use the standard corpus interface
      instead of this method:
  
      Examples
      --------
      .. sourcecode:: pycon
  
          >>> from gensim.test.utils import datapath
          >>> from gensim.corpora import WikiCorpus
          >>>
          >>> path_to_wiki_dump = datapath("enwiki-latest-pages-articles1.xml-p000000010p000030302-shortened.bz2")
          >>>
          >>> for vec in WikiCorpus(path_to_wiki_dump):
          ...     pass
  
      Yields
      ------
      list of str
          If `metadata` is False, yield only list of token extracted from the article.
      (list of str, (int, str))
          List of tokens (extracted from the article), page id and article title otherwise.
  
      """
      articles, articles_all = 0, 0
      positions, positions_all = 0, 0
  
      tokenization_params = (
          self.tokenizer_func,
          self.token_min_len,
          self.token_max_len,
          self.lower,
      )
      texts = (
          (text, title, pageid, tokenization_params)
          for title, text, pageid in extract_pages(
              bz2.BZ2File(self.fname),
              self.filter_namespaces,
              self.filter_articles,
          )
      )
      pool = multiprocessing.Pool(
          self.processes, init_to_ignore_interrupt
      )
  
      try:
          # process the corpus in smaller chunks of docs, because multiprocessing.Pool
          # is dumb and would load the entire input into RAM at once...
          for group in utils.chunkize(
              texts, chunksize=10 * self.processes, maxsize=1
          ):
              for tokens, title, pageid in pool.imap(
                  _process_article, group
              ):
                  articles_all += 1
                  positions_all += len(tokens)
                  # article redirects and short stubs are pruned here
                  if len(tokens) < self.article_min_tokens or any(
                      title.startswith(ignore + ":")
                      for ignore in IGNORED_NAMESPACES
                  ):
                      continue
                  articles += 1
                  positions += len(tokens)
                  if self.metadata:
                      yield (tokens, (pageid, title))
                  else:
                      yield tokens
  
      except KeyboardInterrupt:
          logger.warning(
              "user terminated iteration over Wikipedia corpus after %i documents with %i positions "
              "(total %i articles, %i positions before pruning articles shorter than %i words)",
              articles,
              positions,
              articles_all,
              positions_all,
              self.article_min_tokens,
          )
      except PicklingError as exc:
          raise PicklingError(
              f"Can not send filtering function {self.filter_articles} to multiprocessing, "
              "make sure the function can be pickled."
          ) from exc
      else:
          logger.info(
              "finished iterating over Wikipedia corpus of %i documents with %i positions "
              "(total %i articles, %i positions before pruning articles shorter than %i words)",
              articles,
              positions,
              articles_all,
              positions_all,
              self.article_min_tokens,
          )
          self.length = articles  # cache corpus length
      finally:
          pool.terminate()
  
imports: import bz2
  import logging
  import multiprocessing
  import re
  import signal
  from pickle import PicklingError
  from xml.etree.ElementTree import iterparse
  from gensim import utils
  from gensim.corpora.dictionary import Dictionary
  from gensim.corpora.textcorpus import TextCorpus
label_desc: Use image node_id 3 for calling the WikiCorpus obj's underlying member method code with example usage: obj.get_texts() without return types