





'''simple docstring'''






import os
from shutil import copyfile
from typing import List, Optional, Tuple

from tokenizers import processors

from ...tokenization_utils import AddedToken, BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
      from .tokenization_nllb import NllbTokenizer
else:
      A__							:  int    =None


A__							:  str    =logging.get_logger(__name__)


A__							:  Dict    ={'''vocab_file''': '''sentencepiece.bpe.model''', '''tokenizer_file''': '''tokenizer.json'''}

A__							:  Any    ={
    '''vocab_file''': {
        '''facebook/nllb-200-distilled-600M''': (
            '''https://huggingface.co/facebook/nllb-200-distilled-600M/resolve/main/sentencepiece.bpe.model'''
        ),
    },
    '''tokenizer_file''': {
        '''facebook/nllb-200-distilled-600M''': (
            '''https://huggingface.co/facebook/nllb-200-distilled-600M/resolve/main/tokenizer.json'''
        ),
    },
}

A__							:  List[Any]    ={
    '''facebook/nllb-large-en-ro''': 10_24,
    '''facebook/nllb-200-distilled-600M''': 10_24,
}

# fmt: off
A__							:  List[Any]    =['''ace_Arab''', '''ace_Latn''', '''acm_Arab''', '''acq_Arab''', '''aeb_Arab''', '''afr_Latn''', '''ajp_Arab''', '''aka_Latn''', '''amh_Ethi''', '''apc_Arab''', '''arb_Arab''', '''ars_Arab''', '''ary_Arab''', '''arz_Arab''', '''asm_Beng''', '''ast_Latn''', '''awa_Deva''', '''ayr_Latn''', '''azb_Arab''', '''azj_Latn''', '''bak_Cyrl''', '''bam_Latn''', '''ban_Latn''', '''bel_Cyrl''', '''bem_Latn''', '''ben_Beng''', '''bho_Deva''', '''bjn_Arab''', '''bjn_Latn''', '''bod_Tibt''', '''bos_Latn''', '''bug_Latn''', '''bul_Cyrl''', '''cat_Latn''', '''ceb_Latn''', '''ces_Latn''', '''cjk_Latn''', '''ckb_Arab''', '''crh_Latn''', '''cym_Latn''', '''dan_Latn''', '''deu_Latn''', '''dik_Latn''', '''dyu_Latn''', '''dzo_Tibt''', '''ell_Grek''', '''eng_Latn''', '''epo_Latn''', '''est_Latn''', '''eus_Latn''', '''ewe_Latn''', '''fao_Latn''', '''pes_Arab''', '''fij_Latn''', '''fin_Latn''', '''fon_Latn''', '''fra_Latn''', '''fur_Latn''', '''fuv_Latn''', '''gla_Latn''', '''gle_Latn''', '''glg_Latn''', '''grn_Latn''', '''guj_Gujr''', '''hat_Latn''', '''hau_Latn''', '''heb_Hebr''', '''hin_Deva''', '''hne_Deva''', '''hrv_Latn''', '''hun_Latn''', '''hye_Armn''', '''ibo_Latn''', '''ilo_Latn''', '''ind_Latn''', '''isl_Latn''', '''ita_Latn''', '''jav_Latn''', '''jpn_Jpan''', '''kab_Latn''', '''kac_Latn''', '''kam_Latn''', '''kan_Knda''', '''kas_Arab''', '''kas_Deva''', '''kat_Geor''', '''knc_Arab''', '''knc_Latn''', '''kaz_Cyrl''', '''kbp_Latn''', '''kea_Latn''', '''khm_Khmr''', '''kik_Latn''', '''kin_Latn''', '''kir_Cyrl''', '''kmb_Latn''', '''kon_Latn''', '''kor_Hang''', '''kmr_Latn''', '''lao_Laoo''', '''lvs_Latn''', '''lij_Latn''', '''lim_Latn''', '''lin_Latn''', '''lit_Latn''', '''lmo_Latn''', '''ltg_Latn''', '''ltz_Latn''', '''lua_Latn''', '''lug_Latn''', '''luo_Latn''', '''lus_Latn''', '''mag_Deva''', '''mai_Deva''', '''mal_Mlym''', '''mar_Deva''', '''min_Latn''', '''mkd_Cyrl''', '''plt_Latn''', '''mlt_Latn''', '''mni_Beng''', '''khk_Cyrl''', '''mos_Latn''', '''mri_Latn''', '''zsm_Latn''', '''mya_Mymr''', '''nld_Latn''', '''nno_Latn''', '''nob_Latn''', '''npi_Deva''', '''nso_Latn''', '''nus_Latn''', '''nya_Latn''', '''oci_Latn''', '''gaz_Latn''', '''ory_Orya''', '''pag_Latn''', '''pan_Guru''', '''pap_Latn''', '''pol_Latn''', '''por_Latn''', '''prs_Arab''', '''pbt_Arab''', '''quy_Latn''', '''ron_Latn''', '''run_Latn''', '''rus_Cyrl''', '''sag_Latn''', '''san_Deva''', '''sat_Beng''', '''scn_Latn''', '''shn_Mymr''', '''sin_Sinh''', '''slk_Latn''', '''slv_Latn''', '''smo_Latn''', '''sna_Latn''', '''snd_Arab''', '''som_Latn''', '''sot_Latn''', '''spa_Latn''', '''als_Latn''', '''srd_Latn''', '''srp_Cyrl''', '''ssw_Latn''', '''sun_Latn''', '''swe_Latn''', '''swh_Latn''', '''szl_Latn''', '''tam_Taml''', '''tat_Cyrl''', '''tel_Telu''', '''tgk_Cyrl''', '''tgl_Latn''', '''tha_Thai''', '''tir_Ethi''', '''taq_Latn''', '''taq_Tfng''', '''tpi_Latn''', '''tsn_Latn''', '''tso_Latn''', '''tuk_Latn''', '''tum_Latn''', '''tur_Latn''', '''twi_Latn''', '''tzm_Tfng''', '''uig_Arab''', '''ukr_Cyrl''', '''umb_Latn''', '''urd_Arab''', '''uzn_Latn''', '''vec_Latn''', '''vie_Latn''', '''war_Latn''', '''wol_Latn''', '''xho_Latn''', '''ydd_Hebr''', '''yor_Latn''', '''yue_Hant''', '''zho_Hans''', '''zho_Hant''', '''zul_Latn''']




class 					UpperCAmelCase	(     snake_case_  ):
      _lowercase:     Union[str, Any]			=  VOCAB_FILES_NAMES
      _lowercase:     str			=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      _lowercase:     Tuple			=  PRETRAINED_VOCAB_FILES_MAP
      _lowercase:     Dict			=  ['''input_ids''', '''attention_mask''']
      _lowercase:     List[Any]			=  NllbTokenizer

      _lowercase:     List[int]			=  []
      _lowercase:     List[int]			=  []




      def __init__(     self     :					Union[str, Any]     ,     __snake_case     :					List[Any]=None     ,     __snake_case     :					List[Any]=None     ,     __snake_case     :					Optional[Any]="<s>"     ,     __snake_case     :					Optional[int]="</s>"     ,     __snake_case     :					int="</s>"     ,     __snake_case     :					str="<s>"     ,     __snake_case     :					Tuple="<unk>"     ,     __snake_case     :					List[str]="<pad>"     ,     __snake_case     :					Optional[int]="<mask>"     ,     __snake_case     :					List[Any]=None     ,     __snake_case     :					Any=None     ,     __snake_case     :					Union[str, Any]=None     ,     __snake_case     :					str=False     ,     **__snake_case     :					str     ,     )							-> Any:
            # Mask token behave like a normal word, i.e. include the space before it
            _lowerCAmelCase					=				AddedToken(__snake_case     ,     lstrip=__snake_case     ,     rstrip=__snake_case    ) if isinstance(__snake_case     ,     __snake_case    ) else mask_token
            _lowerCAmelCase					=				legacy_behaviour
            super().__init__(
                vocab_file=__snake_case     ,     tokenizer_file=__snake_case     ,     bos_token=__snake_case     ,     eos_token=__snake_case     ,     sep_token=__snake_case     ,     cls_token=__snake_case     ,     unk_token=__snake_case     ,     pad_token=__snake_case     ,     mask_token=__snake_case     ,     src_lang=__snake_case     ,     tgt_lang=__snake_case     ,     additional_special_tokens=__snake_case     ,     legacy_behaviour=__snake_case     ,     **__snake_case     ,     )

            _lowerCAmelCase					=				vocab_file
            _lowerCAmelCase					=				False if not self.vocab_file else True

            _lowerCAmelCase					=				FAIRSEQ_LANGUAGE_CODES.copy()

            if additional_special_tokens is not None:
                  # Only add those special tokens if they are not already there.
                  _additional_special_tokens.extend(
                      [t for t in additional_special_tokens if t not in _additional_special_tokens]    )

            self.add_special_tokens({"""additional_special_tokens""": _additional_special_tokens}    )
            _lowerCAmelCase					=				{
                lang_code: self.convert_tokens_to_ids(__snake_case    ) for lang_code in FAIRSEQ_LANGUAGE_CODES
            }

            _lowerCAmelCase					=				src_lang if src_lang is not None else """eng_Latn"""
            _lowerCAmelCase					=				self.convert_tokens_to_ids(self._src_lang    )
            _lowerCAmelCase					=				tgt_lang
            self.set_src_lang_special_tokens(self._src_lang    )




      @property
      def 							lowercase__       (     self     :					Tuple    )							-> str:
            return self._src_lang




      @src_lang.setter
      def 							lowercase__       (     self     :					Tuple     ,     __snake_case     :					str    )							-> None:
            _lowerCAmelCase					=				new_src_lang
            self.set_src_lang_special_tokens(self._src_lang    )




      def 							lowercase__       (     self     :					int     ,     __snake_case     :					List[int]     ,     __snake_case     :					Optional[List[int]] = None    )							-> List[int]:
            if token_ids_a is None:
                  return self.prefix_tokens + token_ids_a + self.suffix_tokens
            # We don't expect to process pairs, but leave the pair logic for API consistency
            return self.prefix_tokens + token_ids_a + token_ids_a + self.suffix_tokens




      def 							lowercase__       (     self     :					Optional[Any]     ,     __snake_case     :					List[int]     ,     __snake_case     :					Optional[List[int]] = None    )							-> List[int]:
            _lowerCAmelCase					=				[self.sep_token_id]
            _lowerCAmelCase					=				[self.cls_token_id]

            if token_ids_a is None:
                  return len(cls + token_ids_a + sep    ) * [0]
            return len(cls + token_ids_a + sep + sep + token_ids_a + sep    ) * [0]




      def 							lowercase__       (     self     :					List[Any]     ,     __snake_case     :					Optional[int]     ,     __snake_case     :					str     ,     __snake_case     :					Optional[str]     ,     __snake_case     :					Optional[str]     ,     **__snake_case     :					int    )							-> Any:
            if src_lang is None or tgt_lang is None:
                  raise ValueError("""Translation requires a `src_lang` and a `tgt_lang` for this model"""    )
            _lowerCAmelCase					=				src_lang
            _lowerCAmelCase					=				self(__snake_case     ,     add_special_tokens=__snake_case     ,     return_tensors=__snake_case     ,     **__snake_case    )
            _lowerCAmelCase					=				self.convert_tokens_to_ids(__snake_case    )
            _lowerCAmelCase					=				tgt_lang_id
            return inputs




      def 							lowercase__       (     self     :					Any     ,     __snake_case     :					List[str]     ,     __snake_case     :					str = "eng_Latn"     ,     __snake_case     :					Optional[List[str]] = None     ,     __snake_case     :					str = "fra_Latn"     ,     **__snake_case     :					List[str]     ,     )							-> BatchEncoding:
            _lowerCAmelCase					=				src_lang
            _lowerCAmelCase					=				tgt_lang
            return super().prepare_seqaseq_batch(__snake_case     ,     __snake_case     ,     **__snake_case    )




      def 							lowercase__       (     self     :					str    )							-> str:
            return self.set_src_lang_special_tokens(self.src_lang    )




      def 							lowercase__       (     self     :					Any    )							-> int:
            return self.set_tgt_lang_special_tokens(self.tgt_lang    )




      def 							lowercase__       (     self     :					List[str]     ,     __snake_case     :					str    )							-> None:
            _lowerCAmelCase					=				self.convert_tokens_to_ids(__snake_case    )

            if self.legacy_behaviour:
                  _lowerCAmelCase					=				[]
                  _lowerCAmelCase					=				[self.eos_token_id, self.cur_lang_code]
            else:
                  _lowerCAmelCase					=				[self.cur_lang_code]
                  _lowerCAmelCase					=				[self.eos_token_id]

            _lowerCAmelCase					=				self.convert_ids_to_tokens(self.prefix_tokens    )
            _lowerCAmelCase					=				self.convert_ids_to_tokens(self.suffix_tokens    )

            _lowerCAmelCase					=				processors.TemplateProcessing(
                single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str     ,     pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str     ,     special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str     ,     self.prefix_tokens + self.suffix_tokens    )    )     ,     )




      def 							lowercase__       (     self     :					Optional[int]     ,     __snake_case     :					str    )							-> None:
            _lowerCAmelCase					=				self.convert_tokens_to_ids(__snake_case    )
            if self.legacy_behaviour:
                  _lowerCAmelCase					=				[]
                  _lowerCAmelCase					=				[self.eos_token_id, self.cur_lang_code]
            else:
                  _lowerCAmelCase					=				[self.cur_lang_code]
                  _lowerCAmelCase					=				[self.eos_token_id]

            _lowerCAmelCase					=				self.convert_ids_to_tokens(self.prefix_tokens    )
            _lowerCAmelCase					=				self.convert_ids_to_tokens(self.suffix_tokens    )

            _lowerCAmelCase					=				processors.TemplateProcessing(
                single=prefix_tokens_str + ["""$A"""] + suffix_tokens_str     ,     pair=prefix_tokens_str + ["""$A""", """$B"""] + suffix_tokens_str     ,     special_tokens=list(zip(prefix_tokens_str + suffix_tokens_str     ,     self.prefix_tokens + self.suffix_tokens    )    )     ,     )




      def 							lowercase__       (     self     :					Any     ,     __snake_case     :					str     ,     __snake_case     :					Optional[str] = None    )							-> Tuple[str]:
            if not self.can_save_slow_tokenizer:
                  raise ValueError(
                      """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
                      """tokenizer."""    )

            if not os.path.isdir(__snake_case    ):
                  logger.error(f"Vocabulary path ({save_directory}) should be a directory."    )
                  return
            _lowerCAmelCase					=				os.path.join(
                __snake_case     ,     (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]    )

            if os.path.abspath(self.vocab_file    ) != os.path.abspath(__snake_case    ):
                  copyfile(self.vocab_file     ,     __snake_case    )

            return (out_vocab_file,)













'''simple docstring'''






from typing import List

import jiwer
import jiwer.transforms as tr
from packaging import version

import datasets
from datasets.config import PY_VERSION


if PY_VERSION < version.parse('''3.8'''):
      import importlib_metadata
else:
      import importlib.metadata as importlib_metadata


A__							:  Dict    =''''''


if version.parse(importlib_metadata.version('''jiwer''')) < version.parse('''2.3.0'''):




      class 					UpperCAmelCase	(     tr.AbstractTransform  ):




            def __init__(     self     :					Any     ,     __snake_case     :					str = " "    )							-> Any:
                  _lowerCAmelCase					=				sentence_delimiter




            def 							lowercase__       (     self     :					str     ,     __snake_case     :					str    )							-> Optional[int]:
                  return list(__snake_case    )




            def 							lowercase__       (     self     :					Tuple     ,     __snake_case     :					List[str]    )							-> Union[str, Any]:
                  _lowerCAmelCase					=				[]
                  for sent_idx, sentence in enumerate(__snake_case    ):
                        chars.extend(self.process_string(__snake_case    )    )
                        if self.sentence_delimiter is not None and self.sentence_delimiter != "" and sent_idx < len(__snake_case    ) - 1:
                              chars.append(self.sentence_delimiter    )
                  return chars




      A__							:  Optional[Any]    =tr.Compose(
          [tr.RemoveMultipleSpaces(), tr.Strip(), SentencesToListOfCharacters(SENTENCE_DELIMITER)]
      )
else:
      A__							:  Tuple    =tr.Compose(
          [
              tr.RemoveMultipleSpaces(),
              tr.Strip(),
              tr.ReduceToSingleSentence(SENTENCE_DELIMITER),
              tr.ReduceToListOfListOfChars(),
          ]
      )


A__							:  List[str]    ='''\
@inproceedings{inproceedings,
    author = {Morris, Andrew and Maier, Viktoria and Green, Phil},
    year = {2004},
    month = {01},
    pages = {},
    title = {From WER and RIL to MER and WIL: improved evaluation measures for connected speech recognition.}
}
'''

A__							:  Optional[int]    ='''\
Character error rate (CER) is a common metric of the performance of an automatic speech recognition system.

CER is similar to Word Error Rate (WER), but operates on character instead of word. Please refer to docs of WER for further information.

Character error rate can be computed as:

CER = (S + D + I) / N = (S + D + I) / (S + D + C)

where

S is the number of substitutions,
D is the number of deletions,
I is the number of insertions,
C is the number of correct characters,
N is the number of characters in the reference (N=S+D+C).

CER\'s output is not always a number between 0 and 1, in particular when there is a high number of insertions. This value is often associated to the percentage of characters that were incorrectly predicted. The lower the value, the better the
performance of the ASR system with a CER of 0 being a perfect score.
'''

A__							:  List[str]    ='''
Computes CER score of transcribed segments against references.
Args:
    references: list of references for each speech input.
    predictions: list of transcribtions to score.
    concatenate_texts: Whether or not to concatenate sentences before evaluation, set to True for more accurate result.
Returns:
    (float): the character error rate

Examples:

    >>> predictions = ["this is the prediction", "there is an other sample"]
    >>> references = ["this is the reference", "there is another one"]
    >>> cer = datasets.load_metric("cer")
    >>> cer_score = cer.compute(predictions=predictions, references=references)
    >>> print(cer_score)
    0.34146341463414637
'''




@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION				,						_KWARGS_DESCRIPTION  )
class 					UpperCAmelCase	(     datasets.Metric  ):




      def 							lowercase__       (     self     :					Any    )							-> Dict:
            return datasets.MetricInfo(
                description=_DESCRIPTION     ,     citation=_CITATION     ,     inputs_description=_KWARGS_DESCRIPTION     ,     features=datasets.Features(
                    {
                        """predictions""": datasets.Value("""string"""     ,     id="""sequence"""    ),
                        """references""": datasets.Value("""string"""     ,     id="""sequence"""    ),
                    }    )     ,     codebase_urls=["""https://github.com/jitsi/jiwer/"""]     ,     reference_urls=[
                    """https://en.wikipedia.org/wiki/Word_error_rate""",
                    """https://sites.google.com/site/textdigitisation/qualitymeasures/computingerrorrates""",
                ]     ,     )




      def 							lowercase__       (     self     :					Optional[Any]     ,     __snake_case     :					Optional[Any]     ,     __snake_case     :					List[str]     ,     __snake_case     :					Tuple=False    )							-> List[Any]:
            if concatenate_texts:
                  return jiwer.compute_measures(
                      __snake_case     ,     __snake_case     ,     truth_transform=__snake_case     ,     hypothesis_transform=__snake_case     ,     )["wer"]

            _lowerCAmelCase					=				0
            _lowerCAmelCase					=				0
            for prediction, reference in zip(__snake_case     ,     __snake_case    ):
                  _lowerCAmelCase					=				jiwer.compute_measures(
                      __snake_case     ,     __snake_case     ,     truth_transform=__snake_case     ,     hypothesis_transform=__snake_case     ,     )
                  incorrect += measures["substitutions"] + measures["deletions"] + measures["insertions"]
                  total += measures["substitutions"] + measures["deletions"] + measures["hits"]

            return incorrect / total






