class_docstr: Wikipedia dataset.
class_docstr_tok: ["Wikipedia","dataset","."]
docstr: Build PCollection of examples in the raw (text) form.
docstr_tok: ["Build","PCollection","of","examples","in","the","raw","(","text",")","form","."]
code: def _generate_examples(self, filepaths, language):
      """Build PCollection of examples in the raw (text) form."""
      beam = tfds.core.lazy_imports.apache_beam
      mwparserfromhell = tfds.core.lazy_imports.mwparserfromhell
      mwxml = tfds.core.lazy_imports.mwxml
  
      def _extract_content(filepath):
          """Extracts article content from a single WikiMedia XML file."""
          filepath = epath.Path(filepath)
          logging.info("generating examples from = %s", filepath)
          with filepath.open("rb") as f:
              f = bz2.BZ2File(filename=f)
              dump = mwxml.Dump.from_file(f)
              for page in dump:
                  beam.metrics.Metrics.counter(
                      language, "pages-read"
                  ).inc()
                  if page is None:
                      continue
                  # Filter pages that are not in the "main" namespace.
                  if page.namespace != 0:
                      continue
                  beam.metrics.Metrics.counter(
                      language, "pages-in-main-read"
                  ).inc()
  
                  try:
                      revision = next(
                          iter(page)
                      )  # A single revision in dumps.
                  except (TypeError, StopIteration):
                      beam.metrics.Metrics.counter(
                          language, "no-revision-error"
                      ).inc()
                      continue
  
                  if not revision.text:
                      beam.metrics.Metrics.counter(
                          language, "empty-revision-error"
                      ).inc()
                      continue
  
                  if revision.text[:9].lower().startswith("#redirect"):
                      beam.metrics.Metrics.counter(
                          language, "filtered-redirects"
                      ).inc()
                      continue
  
                  yield WikiPage(
                      id=page.id, title=page.title, text=revision.text
                  )
  
      # Filters for references, tables, and file/image links.
      re_rm_wikilink = re.compile(
          "^(?:File|Image|Media):", flags=re.IGNORECASE | re.UNICODE
      )
  
      def _parse_and_clean_wikicode(raw_content):
          """Strips formatting and unwanted sections from raw page content."""
          wikicode = mwparserfromhell.parse(raw_content)
  
          def rm_wikilink(obj):
              return bool(
                  re_rm_wikilink.match(str(obj.title))
              )  # pytype: disable=wrong-arg-types
  
          def rm_tag(obj):
              return str(obj.tag) in {"ref", "table"}
  
          def rm_template(obj):
              return obj.name.lower() in {
                  "reflist",
                  "notelist",
                  "notelist-ua",
                  "notelist-lr",
                  "notelist-ur",
                  "notelist-lg",
              }
  
          def try_remove_obj(obj, section):
              try:
                  section.remove(obj)
              except ValueError:
                  # For unknown reasons, objects are sometimes not found.
                  pass
  
          section_text = []
          # Filter individual sections to clean.
          for section in wikicode.get_sections(
              flat=True, include_lead=True, include_headings=True
          ):
              for obj in section.ifilter_wikilinks(
                  matches=rm_wikilink, recursive=True
              ):
                  try_remove_obj(obj, section)
              for obj in section.ifilter_templates(
                  matches=rm_template, recursive=True
              ):
                  try_remove_obj(obj, section)
              for obj in section.ifilter_tags(
                  matches=rm_tag, recursive=True
              ):
                  try_remove_obj(obj, section)
  
              section_text.append(section.strip_code().strip())
          return "\n\n".join(section_text)
  
      def _parse_page(page: WikiPage):
          beam.metrics.Metrics.counter(
              language, "extracted-examples"
          ).inc()
          try:
              text = _parse_and_clean_wikicode(page.text)
          except (
              tfds.core.lazy_imports.mwparserfromhell.parser.ParserError
          ) as e:
              beam.metrics.Metrics.counter(
                  language, "parser-error"
              ).inc()
              logging.error("mwparserfromhell ParseError: %s", e)
              return
  
          if not text:
              beam.metrics.Metrics.counter(
                  language, "empty-clean-examples"
              ).inc()
              return
  
          beam.metrics.Metrics.counter(
              language, "cleaned-examples"
          ).inc()
  
          yield page.id, {
              "title": page.title.replace("_", " "),
              "text": text,
          }
  
      return (
          beam.Create(filepaths)
          | beam.FlatMap(_extract_content)
          | beam.Reshuffle()
          | beam.FlatMap(_parse_page)
      )
  
file_path: datasets/tensorflow_datasets/text/wikipedia.py
imports: import bz2
  import dataclasses
  import json
  import re
  from absl import flags
  from absl import logging
  from etils import epath
  import tensorflow_datasets.public_api
label_desc: Use image node_id 3 for calling the Wikipedia obj's underlying member method code with example usage: obj._generate_examples(filepaths, language) and returns: unknown, str, bool, unknown, unknown