





from typing import List

import jiwer
import jiwer.transforms as tr
from packaging import version

import datasets
from datasets.config import PY_VERSION


if PY_VERSION < version.parse('3.8'):
      import importlib_metadata
else:
      import importlib.metadata as importlib_metadata


a__      :     Any         =		""


if version.parse(importlib_metadata.version('jiwer')) < version.parse('2.3.0'):





      class UpperCAmelCase_				( tr.AbstractTransform ):


        def __init__(							self ,__snake_case = " " ):



             """simple docstring"""





             A_          =       sentence_delimiter


        def 						__UpperCAmelCase     (							self ,__snake_case ):



             """simple docstring"""





             return list(__snake_case )




        def 						__UpperCAmelCase     (							self ,__snake_case ):



             """simple docstring"""





             A_          =       []
             for sent_idx, sentence in enumerate(__snake_case ):
                  chars.extend(self.process_string(__snake_case ) )
                  if self.sentence_delimiter is not None and self.sentence_delimiter != "" and sent_idx < len(__snake_case ) - 1:
                       chars.append(self.sentence_delimiter )
             return chars




      a__      :     int         =		tr.Compose(
          [tr.RemoveMultipleSpaces(), tr.Strip(), SentencesToListOfCharacters(SENTENCE_DELIMITER)]
      )
else:
      a__      :     List[str]         =		tr.Compose(
          [
              tr.RemoveMultipleSpaces(),
              tr.Strip(),
              tr.ReduceToSingleSentence(SENTENCE_DELIMITER),
              tr.ReduceToListOfListOfChars(),
          ]
      )


a__      :     Optional[Any]         =		"\\n@inproceedings{inproceedings,\n    author = {Morris, Andrew and Maier, Viktoria and Green, Phil},\n    year = {2004},\n    month = {01},\n    pages = {},\n    title = {From WER and RIL to MER and WIL: improved evaluation measures for connected speech recognition.}\n}\n"

a__      :     List[str]         =		"\\nCharacter error rate (CER) is a common metric of the performance of an automatic speech recognition system.\n\nCER is similar to Word Error Rate (WER), but operates on character instead of word. Please refer to docs of WER for further information.\n\nCharacter error rate can be computed as:\n\nCER = (S + D + I) / N = (S + D + I) / (S + D + C)\n\nwhere\n\nS is the number of substitutions,\nD is the number of deletions,\nI is the number of insertions,\nC is the number of correct characters,\nN is the number of characters in the reference (N=S+D+C).\n\nCER's output is not always a number between 0 and 1, in particular when there is a high number of insertions. This value is often associated to the percentage of characters that were incorrectly predicted. The lower the value, the better the\nperformance of the ASR system with a CER of 0 being a perfect score.\n"

a__      :     int         =		"\nComputes CER score of transcribed segments against references.\nArgs:\n    references: list of references for each speech input.\n    predictions: list of transcribtions to score.\n    concatenate_texts: Whether or not to concatenate sentences before evaluation, set to True for more accurate result.\nReturns:\n    (float): the character error rate\n\nExamples:\n\n    >>> predictions = [\"this is the prediction\", \"there is an other sample\"]\n    >>> references = [\"this is the reference\", \"there is another one\"]\n    >>> cer = datasets.load_metric(\"cer\")\n    >>> cer_score = cer.compute(predictions=predictions, references=references)\n    >>> print(cer_score)\n    0.34146341463414637\n"





@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION     ,		_KWARGS_DESCRIPTION )
class UpperCAmelCase_				( datasets.Metric ):


  def 						__UpperCAmelCase     (							self ):



       """simple docstring"""





       return datasets.MetricInfo(
           description=_DESCRIPTION ,citation=_CITATION ,inputs_description=_KWARGS_DESCRIPTION ,features=datasets.Features(
               {
                   '''predictions''': datasets.Value('''string''' ,id='''sequence''' ),
                   '''references''': datasets.Value('''string''' ,id='''sequence''' ),
               } ) ,codebase_urls=['''https://github.com/jitsi/jiwer/'''] ,reference_urls=[
               '''https://en.wikipedia.org/wiki/Word_error_rate''',
               '''https://sites.google.com/site/textdigitisation/qualitymeasures/computingerrorrates''',
           ] ,)




  def 						__UpperCAmelCase     (							self ,__snake_case ,__snake_case ,__snake_case=False ):



       """simple docstring"""





       if concatenate_texts:
            return jiwer.compute_measures(
                __snake_case ,__snake_case ,truth_transform=__snake_case ,hypothesis_transform=__snake_case ,)["wer"]

       A_          =       0
       A_          =       0
       for prediction, reference in zip(__snake_case ,__snake_case ):
            A_          =       jiwer.compute_measures(
                __snake_case ,__snake_case ,truth_transform=__snake_case ,hypothesis_transform=__snake_case ,)
            incorrect += measures["substitutions"] + measures["deletions"] + measures["insertions"]
            total += measures["substitutions"] + measures["deletions"] + measures["hits"]

       return incorrect / total







import json
import os
from functools import lru_cache
from typing import List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_SCREAMING_SNAKE_CASE     :					Any      =       logging.get_logger(__name__)


_SCREAMING_SNAKE_CASE     :					str      =       {"vocab_file": "vocab.json", "merges_file": "merges.txt"}

_SCREAMING_SNAKE_CASE     :					Any      =       {
    "vocab_file": {
        "allenai/longformer-base-4096": "https://huggingface.co/allenai/longformer-base-4096/resolve/main/vocab.json",
        "allenai/longformer-large-4096": (
            "https://huggingface.co/allenai/longformer-large-4096/resolve/main/vocab.json"
        ),
        "allenai/longformer-large-4096-finetuned-triviaqa": (
            "https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/vocab.json"
        ),
        "allenai/longformer-base-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/vocab.json"
        ),
        "allenai/longformer-large-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/vocab.json"
        ),
    },
    "merges_file": {
        "allenai/longformer-base-4096": "https://huggingface.co/allenai/longformer-base-4096/resolve/main/merges.txt",
        "allenai/longformer-large-4096": (
            "https://huggingface.co/allenai/longformer-large-4096/resolve/main/merges.txt"
        ),
        "allenai/longformer-large-4096-finetuned-triviaqa": (
            "https://huggingface.co/allenai/longformer-large-4096-finetuned-triviaqa/resolve/main/merges.txt"
        ),
        "allenai/longformer-base-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-base-4096-extra.pos.embd.only/resolve/main/merges.txt"
        ),
        "allenai/longformer-large-4096-extra.pos.embd.only": (
            "https://huggingface.co/allenai/longformer-large-4096-extra.pos.embd.only/resolve/main/merges.txt"
        ),
    },
}

_SCREAMING_SNAKE_CASE     :					List[Any]      =       {
    "allenai/longformer-base-4096": 40_96,
    "allenai/longformer-large-4096": 40_96,
    "allenai/longformer-large-4096-finetuned-triviaqa": 40_96,
    "allenai/longformer-base-4096-extra.pos.embd.only": 40_96,
    "allenai/longformer-large-4096-extra.pos.embd.only": 40_96,
}
@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def      UpperCAmelCase__		():



     """simple docstring"""






     snake_case											=							(
         list(range(ord('''!'''							)				,ord('''~'''							) + 1							)							) + list(range(ord('''¡'''							)				,ord('''¬'''							) + 1							)							) + list(range(ord('''®'''							)				,ord('''ÿ'''							) + 1							)							)
     )
     snake_case											=							bs[:]
     snake_case											=							0
     for b in range(2**8							):
          if b not in bs:
               bs.append(UpperCamelCase_							)
               cs.append(2**8 + n							)
               n += 1
     snake_case											=							[chr(UpperCamelCase_							) for n in cs]
     return dict(zip(UpperCamelCase_				,UpperCamelCase_							)							)
def      UpperCAmelCase__		(UpperCamelCase_							):



     """simple docstring"""






     snake_case											=							set()
     snake_case											=							word[0]
     for char in word[1:]:
          pairs.add((prev_char, char)							)
          snake_case											=							char
     return pairs

class     A__   (       snake_case__      ):







    """simple docstring"""

    __magic_name__         =						VOCAB_FILES_NAMES
    __magic_name__         =						PRETRAINED_VOCAB_FILES_MAP
    __magic_name__         =						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    __magic_name__         =						['input_ids', 'attention_mask']
    def __init__(			self     ,   __snake_case     ,   __snake_case     ,   __snake_case="replace"     ,   __snake_case="<s>"     ,   __snake_case="</s>"     ,   __snake_case="</s>"     ,   __snake_case="<s>"     ,   __snake_case="<unk>"     ,   __snake_case="<pad>"     ,   __snake_case="<mask>"     ,   __snake_case=False     ,   **__snake_case     ,   ):
         snake_case											=							AddedToken(__snake_case     ,   lstrip=__snake_case     ,   rstrip=__snake_case       ) if isinstance(__snake_case     ,   __snake_case       ) else bos_token
         snake_case											=							AddedToken(__snake_case     ,   lstrip=__snake_case     ,   rstrip=__snake_case       ) if isinstance(__snake_case     ,   __snake_case       ) else eos_token
         snake_case											=							AddedToken(__snake_case     ,   lstrip=__snake_case     ,   rstrip=__snake_case       ) if isinstance(__snake_case     ,   __snake_case       ) else sep_token
         snake_case											=							AddedToken(__snake_case     ,   lstrip=__snake_case     ,   rstrip=__snake_case       ) if isinstance(__snake_case     ,   __snake_case       ) else cls_token
         snake_case											=							AddedToken(__snake_case     ,   lstrip=__snake_case     ,   rstrip=__snake_case       ) if isinstance(__snake_case     ,   __snake_case       ) else unk_token
         snake_case											=							AddedToken(__snake_case     ,   lstrip=__snake_case     ,   rstrip=__snake_case       ) if isinstance(__snake_case     ,   __snake_case       ) else pad_token

         # Mask token behave like a normal word, i.e. include the space before it
         snake_case											=							AddedToken(__snake_case     ,   lstrip=__snake_case     ,   rstrip=__snake_case       ) if isinstance(__snake_case     ,   __snake_case       ) else mask_token

         super().__init__(
             errors=__snake_case     ,   bos_token=__snake_case     ,   eos_token=__snake_case     ,   unk_token=__snake_case     ,   sep_token=__snake_case     ,   cls_token=__snake_case     ,   pad_token=__snake_case     ,   mask_token=__snake_case     ,   add_prefix_space=__snake_case     ,   **__snake_case     ,   )

         with open(__snake_case     ,   encoding='''utf-8'''       ) as vocab_handle:
              snake_case											=							json.load(__snake_case       )
         snake_case											=							{v: k for k, v in self.encoder.items()}
         snake_case											=							errors  # how to handle errors in decoding
         snake_case											=							bytes_to_unicode()
         snake_case											=							{v: k for k, v in self.byte_encoder.items()}
         with open(__snake_case     ,   encoding='''utf-8'''       ) as merges_handle:
              snake_case											=							merges_handle.read().split('''\n'''       )[1:-1]
         snake_case											=							[tuple(merge.split()       ) for merge in bpe_merges]
         snake_case											=							dict(zip(__snake_case     ,   range(len(__snake_case       )       )       )       )
         snake_case											=							{}
         snake_case											=							add_prefix_space

         # Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
         snake_case											=							re.compile(R'''\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'''       )
    @property
    def 						a_     (			self       ):
         return len(self.encoder       )
    def 						a_     (			self       ):
         return dict(self.encoder     ,   **self.added_tokens_encoder       )
    def 						a_     (			self     ,   __snake_case       ):
         if token in self.cache:
              return self.cache[token]
         snake_case											=							tuple(__snake_case       )
         snake_case											=							get_pairs(__snake_case       )

         if not pairs:
              return token

         while True:
              snake_case											=							min(__snake_case     ,   key=lambda __snake_case       : self.bpe_ranks.get(__snake_case     ,   float('''inf'''       )       )       )
              if bigram not in self.bpe_ranks:
                   break
              snake_case					,    snake_case											=							bigram
              snake_case											=							[]
              snake_case											=							0
              while i < len(__snake_case       ):
                   try:
                        snake_case											=							word.index(__snake_case     ,   __snake_case       )
                   except ValueError:
                        new_word.extend(word[i:]       )
                        break
                   else:
                        new_word.extend(word[i:j]       )
                        snake_case											=							j

                   if word[i] == first and i < len(__snake_case       ) - 1 and word[i + 1] == second:
                        new_word.append(first + second       )
                        i += 2
                   else:
                        new_word.append(word[i]       )
                        i += 1
              snake_case											=							tuple(__snake_case       )
              snake_case											=							new_word
              if len(__snake_case       ) == 1:
                   break
              else:
                   snake_case											=							get_pairs(__snake_case       )
         snake_case											=							''' '''.join(__snake_case       )
         snake_case											=							word
         return word
    def 						a_     (			self     ,   __snake_case       ):
         snake_case											=							[]
         for token in re.findall(self.pat     ,   __snake_case       ):
              snake_case											=							''''''.join(
                  self.byte_encoder[b] for b in token.encode('''utf-8'''       )       )  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
              bpe_tokens.extend(bpe_token for bpe_token in self.bpe(__snake_case       ).split(''' '''       )       )
         return bpe_tokens
    def 						a_     (			self     ,   __snake_case       ):
         return self.encoder.get(__snake_case     ,   self.encoder.get(self.unk_token       )       )
    def 						a_     (			self     ,   __snake_case       ):
         return self.decoder.get(__snake_case       )
    def 						a_     (			self     ,   __snake_case       ):
         snake_case											=							''''''.join(__snake_case       )
         snake_case											=							bytearray([self.byte_decoder[c] for c in text]       ).decode('''utf-8'''     ,   errors=self.errors       )
         return text
    def 						a_     (			self     ,   __snake_case     ,   __snake_case = None       ):
         if not os.path.isdir(__snake_case       ):
              logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''       )
              return
         snake_case											=							os.path.join(
             __snake_case     ,   (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']       )
         snake_case											=							os.path.join(
             __snake_case     ,   (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''merges_file''']       )

         with open(__snake_case     ,   '''w'''     ,   encoding='''utf-8'''       ) as f:
              f.write(json.dumps(self.encoder     ,   indent=2     ,   sort_keys=__snake_case     ,   ensure_ascii=__snake_case       ) + '''\n'''       )

         snake_case											=							0
         with open(__snake_case     ,   '''w'''     ,   encoding='''utf-8'''       ) as writer:
              writer.write('''#version: 0.2\n'''       )
              for bpe_tokens, token_index in sorted(self.bpe_ranks.items()     ,   key=lambda __snake_case       : kv[1]       ):
                   if index != token_index:
                        logger.warning(
                            F'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
                            ''' Please check that the tokenizer is not corrupted!'''       )
                        snake_case											=							token_index
                   writer.write(''' '''.join(__snake_case       ) + '''\n'''       )
                   index += 1

         return vocab_file, merge_file
    def 						a_     (			self     ,   __snake_case     ,   __snake_case = None       ):
         if token_ids_a is None:
              return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
         snake_case											=							[self.cls_token_id]
         snake_case											=							[self.sep_token_id]
         return cls + token_ids_a + sep + sep + token_ids_a + sep
    def 						a_     (			self     ,   __snake_case     ,   __snake_case = None     ,   __snake_case = False       ):
         if already_has_special_tokens:
              return super().get_special_tokens_mask(
                  token_ids_a=__snake_case     ,   token_ids_a=__snake_case     ,   already_has_special_tokens=__snake_case       )

         if token_ids_a is None:
              return [1] + ([0] * len(__snake_case       )) + [1]
         return [1] + ([0] * len(__snake_case       )) + [1, 1] + ([0] * len(__snake_case       )) + [1]
    def 						a_     (			self     ,   __snake_case     ,   __snake_case = None       ):
         snake_case											=							[self.sep_token_id]
         snake_case											=							[self.cls_token_id]

         if token_ids_a is None:
              return len(cls + token_ids_a + sep       ) * [0]
         return len(cls + token_ids_a + sep + sep + token_ids_a + sep       ) * [0]




    def 						a_     (			self     ,   __snake_case     ,   __snake_case=False     ,   **__snake_case       ):
         snake_case											=							kwargs.pop('''add_prefix_space'''     ,   self.add_prefix_space       )
         if (is_split_into_words or add_prefix_space) and (len(__snake_case       ) > 0 and not text[0].isspace()):
              snake_case											=							''' ''' + text
         return (text, kwargs)




