code: def extract_snippets(
      c4_doc: Mapping[str, bytes],
      max_sentences_per_snippet: int,
      abbreviations_by_expansion: Mapping[str, Sequence[str]],
      expansion_regex: re.Pattern[str],
      max_snippet_char_len: int,
      alpha_keep_no_rs: float,
      alpha_keep_rs: float,
  ) -> Iterator[
      tuple[
          str,
          tuple[
              str,
              AbbrevExpansionPair,
              Mapping[int, AbbrevExpansionPair],
          ],
      ]
  ]:
      """Extracts variable-length multi-sentence snippets from C4 web text.
  
      Determines whether to keep or discard snippets based on the number of snippets
      previously kept which contain the same reverse substitution candidates (or
      lack thereof).
  
      Args:
        c4_doc: The C4 web crawl document.
        max_sentences_per_snippet: The maximum number of sentences that can be
          combined into a single snippet. The number of sentences combined into each
          snippet will be a randomly sampled integer from 1 to
          max_sentences_per_snippet.
        abbreviations_by_expansion: A mapping from expansion to all valid
          abbreviations for that expansion.
        expansion_regex: A regex pattern that will match any valid expansion.
        max_snippet_char_len: The maximum number of characters an original snippet
          can be composed of.
        alpha_keep_no_rs: Alpha value used to compute the probability of keeping a
          snippet containing no reverse substitutions. The complete formula is p =
          (n+1)**-alpha where n is the count of previously kept snippets from the
          same document.
        alpha_keep_rs: Alpha value used to compute the probability of keeping a
          snippet containing a particular abbreviation-expansion reverse
          substitution candidate. The complete formula is p = (n+1)**-alpha where n
          is the count of previously kept snippets from the same document containing
          the same reverse substitution candidate.
  
      Yields:
        A key-value tuple where the key is a string containing the document url and
        incremental snippet id, and the value is a 3-tuple containing the snippet,
        the rarest abbreviation-expansion substitution candidate in the snippet, and
        a mapping from snippet index to each substitution candidate.
      """
      count_by_pair = collections.Counter()
      text = c4_doc["text"].decode()
      text = text.replace("\n", " ")
      sentences = text.split(". ")
      sentence_idx = 0
      snippet_id = 0
      while sentence_idx < len(sentences):
          num_sentences_sampled = np.random.randint(
              1, max_sentences_per_snippet + 1
          )
          snippet = ". ".join(
              sentences[
                  sentence_idx : sentence_idx + num_sentences_sampled
              ]
          )
          # Period added stochastically to make models robust to the absence of
          # ending punctuation.
          if not snippet.endswith(".") and np.random.uniform() < 0.5:
              snippet += "."
          key = f'url={c4_doc["url"].decode()},snippet_id={snippet_id}'
          snippet_id += 1
          sentence_idx += num_sentences_sampled
          snippet = snippet.lower().strip()
          if len(snippet) > max_snippet_char_len:
              continue
          index_to_pair = _get_abbreviation_expansion_pairs(
              snippet, abbreviations_by_expansion, expansion_regex
          )
          if index_to_pair:
              smallest_count, rarest_pair = None, None
              for pair in index_to_pair.values():
                  if pair[1] in FREQUENT_EXPANSIONS:
                      continue
                  count = count_by_pair[pair]
                  if smallest_count is None or count < smallest_count:
                      smallest_count = count
                      rarest_pair = pair
              sample_prob = (
                  _get_backoff_prob(smallest_count, alpha=alpha_keep_rs)
                  if smallest_count is not None
                  else 0.0
              )
              if np.random.uniform() >= sample_prob:
                  continue
              # Count should only be increased if snippet is sampled.
              for pair in index_to_pair.values():
                  count_by_pair[pair] += 1
          else:
              rarest_pair = ("", "")
              index_to_pair = {0: ("", "")}
              count = count_by_pair[("", "")]
              sample_prob = _get_backoff_prob(
                  count + 100, alpha=alpha_keep_no_rs
              )
              if np.random.uniform() >= sample_prob:
                  continue
              count_by_pair[("", "")] += 1
  
          yield key, (snippet, rarest_pair, index_to_pair)
  
file_path: datasets/tensorflow_datasets/text/c4_wsrs/c4_wsrs_utils.py
imports: from __future__ import annotations
  import collections
  import dataclasses
  import random
  import re
  from typing import Iterator, Mapping, Sequence, Tuple
  import numpy
label_desc: Use image node_id 4 for calling a global function with example usage: extract_snippets(c4_doc, max_sentences_per_snippet, abbreviations_by_expansion, expansion_regex, max_snippet_char_len, alpha_keep_no_rs, alpha_keep_rs) without return types