



from typing import List

import jiwer
import jiwer.transforms as tr
from packaging import version

import datasets
from datasets.config import PY_VERSION


if PY_VERSION < version.parse("3.8"):
			import importlib_metadata
else:
			import importlib.metadata as importlib_metadata


SCREAMING_SNAKE_CASE  					=    ''


if version.parse(importlib_metadata.version("jiwer")) < version.parse("2.3.0"):



			class 						lowerCamelCase   (    tr.AbstractTransform	):







								'''simple docstring'''







								def __init__(      self     ,						lowerCAmelCase = " "  ):
										UpperCAmelCase_						       =					sentence_delimiter



								def  A__		(      self     ,						lowerCAmelCase  ):
										return list(A__  )



								def  A__		(      self     ,						lowerCAmelCase  ):
										UpperCAmelCase_						       =					[]
										for sent_idx, sentence in enumerate(A__  ):
												chars.extend(self.process_string(A__  )  )
												if self.sentence_delimiter is not None and self.sentence_delimiter != "" and sent_idx < len(A__  ) - 1:
														chars.append(self.sentence_delimiter  )
										return chars





			SCREAMING_SNAKE_CASE  					=    tr.Compose(
			    [tr.RemoveMultipleSpaces(), tr.Strip(), SentencesToListOfCharacters(SENTENCE_DELIMITER)]
			)
else:
			SCREAMING_SNAKE_CASE  					=    tr.Compose(
			    [
			        tr.RemoveMultipleSpaces(),
			        tr.Strip(),
			        tr.ReduceToSingleSentence(SENTENCE_DELIMITER),
			        tr.ReduceToListOfListOfChars(),
			    ]
			)


SCREAMING_SNAKE_CASE  					=    '\\n@inproceedings{inproceedings,\n    author = {Morris, Andrew and Maier, Viktoria and Green, Phil},\n    year = {2004},\n    month = {01},\n    pages = {},\n    title = {From WER and RIL to MER and WIL: improved evaluation measures for connected speech recognition.}\n}\n'

SCREAMING_SNAKE_CASE  					=    '\\nCharacter error rate (CER) is a common metric of the performance of an automatic speech recognition system.\n\nCER is similar to Word Error Rate (WER), but operates on character instead of word. Please refer to docs of WER for further information.\n\nCharacter error rate can be computed as:\n\nCER = (S + D + I) / N = (S + D + I) / (S + D + C)\n\nwhere\n\nS is the number of substitutions,\nD is the number of deletions,\nI is the number of insertions,\nC is the number of correct characters,\nN is the number of characters in the reference (N=S+D+C).\n\nCER\'s output is not always a number between 0 and 1, in particular when there is a high number of insertions. This value is often associated to the percentage of characters that were incorrectly predicted. The lower the value, the better the\nperformance of the ASR system with a CER of 0 being a perfect score.\n'

SCREAMING_SNAKE_CASE  					=    '\nComputes CER score of transcribed segments against references.\nArgs:\n    references: list of references for each speech input.\n    predictions: list of transcribtions to score.\n    concatenate_texts: Whether or not to concatenate sentences before evaluation, set to True for more accurate result.\nReturns:\n    (float): the character error rate\n\nExamples:\n\n    >>> predictions = ["this is the prediction", "there is an other sample"]\n    >>> references = ["this is the reference", "there is another one"]\n    >>> cer = datasets.load_metric("cer")\n    >>> cer_score = cer.compute(predictions=predictions, references=references)\n    >>> print(cer_score)\n    0.34146341463414637\n'



@datasets.utils.file_utils.add_start_docstrings(_DESCRIPTION,  _KWARGS_DESCRIPTION	)
class 						lowerCamelCase   (    datasets.Metric	):







					'''simple docstring'''







					def  A__		(      self  ):
							return datasets.MetricInfo(
							    description=_DESCRIPTION     ,						citation=_CITATION     ,						inputs_description=_KWARGS_DESCRIPTION     ,						features=datasets.Features(
							        {
							            "predictions": datasets.Value("string"     ,						id="sequence"  ),
							            "references": datasets.Value("string"     ,						id="sequence"  ),
							        }  )     ,						codebase_urls=["https://github.com/jitsi/jiwer/"]     ,						reference_urls=[
							        "https://en.wikipedia.org/wiki/Word_error_rate",
							        "https://sites.google.com/site/textdigitisation/qualitymeasures/computingerrorrates",
							    ]     ,						)



					def  A__		(      self     ,						lowerCAmelCase     ,						lowerCAmelCase     ,						lowerCAmelCase=False  ):
							if concatenate_texts:
									return jiwer.compute_measures(
									    A__     ,						A__     ,						truth_transform=A__     ,						hypothesis_transform=A__     ,						)["wer"]

							UpperCAmelCase_						       =					0
							UpperCAmelCase_						       =					0
							for prediction, reference in zip(A__     ,						A__  ):
									UpperCAmelCase_						       =					jiwer.compute_measures(
									    A__     ,						A__     ,						truth_transform=A__     ,						hypothesis_transform=A__     ,						)
									incorrect += measures["substitutions"] + measures["deletions"] + measures["insertions"]
									total += measures["substitutions"] + measures["deletions"] + measures["hits"]

							return incorrect / total









import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


_UpperCamelCase			:	Dict								=get_tests_dir('fixtures/test_sentencepiece.model')


if is_torch_available():
	from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

_UpperCamelCase			:	Optional[Any]								=256047
_UpperCamelCase			:	int								=256145






@require_sentencepiece
@require_tokenizers
class 						UpperCAmelCase__ (							__snake_case     ,				unittest.TestCase							):
		__snake_case      :       Any     =   NllbTokenizer
		__snake_case      :       Dict     =   NllbTokenizerFast
		__snake_case      :       List[Any]     =   True
		__snake_case      :       Any     =   True
		__snake_case      :       List[str]     =   {}







		def  A__      (					self		):
						super().setUp()

						# We have a SentencePiece fixture for testing
						_A	:				Any   =      NllbTokenizer(A__       ,keep_accents=A__		)
						tokenizer.save_pretrained(self.tmpdirname		)







		def  A__      (					self		):
						_A	:				Dict   =      NllbTokenizer(A__       ,keep_accents=A__		)

						_A	:				Any   =      tokenizer.tokenize('''This is a test'''		)
						self.assertListEqual(A__       ,['''▁This''', '''▁is''', '''▁a''', '''▁t''', '''est''']		)

						self.assertListEqual(
						    tokenizer.convert_tokens_to_ids(A__		)       ,[value + tokenizer.fairseq_offset for value in [285, 46, 10, 170, 382]]       ,)

						_A	:				Optional[Any]   =      tokenizer.tokenize('''I was born in 92000, and this is falsé.'''		)
						self.assertListEqual(
						    A__       ,[
						        SPIECE_UNDERLINE + '''I''',
						        SPIECE_UNDERLINE + '''was''',
						        SPIECE_UNDERLINE + '''b''',
						        '''or''',
						        '''n''',
						        SPIECE_UNDERLINE + '''in''',
						        SPIECE_UNDERLINE + '''''',
						        '''9''',
						        '''2''',
						        '''0''',
						        '''0''',
						        '''0''',
						        ''',''',
						        SPIECE_UNDERLINE + '''and''',
						        SPIECE_UNDERLINE + '''this''',
						        SPIECE_UNDERLINE + '''is''',
						        SPIECE_UNDERLINE + '''f''',
						        '''al''',
						        '''s''',
						        '''é''',
						        '''.''',
						    ]       ,)
						_A	:				int   =      tokenizer.convert_tokens_to_ids(A__		)
						self.assertListEqual(
						    A__       ,[
						        value + tokenizer.fairseq_offset
						        for value in [8, 21, 84, 55, 24, 19, 7, 2, 602, 347, 347, 347, 3, 12, 66, 46, 72, 80, 6, 2, 4]
						    ]       ,)

						_A	:				Optional[int]   =      tokenizer.convert_ids_to_tokens(A__		)
						self.assertListEqual(
						    A__       ,[
						        SPIECE_UNDERLINE + '''I''',
						        SPIECE_UNDERLINE + '''was''',
						        SPIECE_UNDERLINE + '''b''',
						        '''or''',
						        '''n''',
						        SPIECE_UNDERLINE + '''in''',
						        SPIECE_UNDERLINE + '''''',
						        '''<unk>''',
						        '''2''',
						        '''0''',
						        '''0''',
						        '''0''',
						        ''',''',
						        SPIECE_UNDERLINE + '''and''',
						        SPIECE_UNDERLINE + '''this''',
						        SPIECE_UNDERLINE + '''is''',
						        SPIECE_UNDERLINE + '''f''',
						        '''al''',
						        '''s''',
						        '''<unk>''',
						        '''.''',
						    ]       ,)







		def  A__      (					self		):
						_A	:				Any   =      (self.rust_tokenizer_class, '''hf-internal-testing/tiny-random-nllb''', {})
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""		):
														_A	:				List[str]   =      self.rust_tokenizer_class.from_pretrained(A__       ,**A__		)
														_A	:				Tuple   =      self.tokenizer_class.from_pretrained(A__       ,**A__		)

														_A	:				Tuple   =      tempfile.mkdtemp()

														_A	:				List[Any]   =      tokenizer_r.save_pretrained(A__		)
														_A	:				Tuple   =      tokenizer_p.save_pretrained(A__		)

														# Checks it save with the same files + the tokenizer.json file for the fast one
														self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files		)		)
														_A	:				Dict   =      tuple(f for f in tokenizer_r_files if '''tokenizer.json''' not in f		)
														self.assertSequenceEqual(A__       ,A__		)

														# Checks everything loads correctly in the same way
														_A	:				Optional[int]   =      tokenizer_r.from_pretrained(A__		)
														_A	:				Optional[Any]   =      tokenizer_p.from_pretrained(A__		)

														# Check special tokens are set accordingly on Rust and Python
														for key in tokenizer_pp.special_tokens_map:
																		self.assertTrue(hasattr(A__       ,A__		)		)

														shutil.rmtree(A__		)

														# Save tokenizer rust, legacy_format=True
														_A	:				List[Any]   =      tempfile.mkdtemp()

														_A	:				List[Any]   =      tokenizer_r.save_pretrained(A__       ,legacy_format=A__		)
														_A	:				List[Any]   =      tokenizer_p.save_pretrained(A__		)

														# Checks it save with the same files
														self.assertSequenceEqual(A__       ,A__		)

														# Checks everything loads correctly in the same way
														_A	:				List[Any]   =      tokenizer_r.from_pretrained(A__		)
														_A	:				int   =      tokenizer_p.from_pretrained(A__		)

														# Check special tokens are set accordingly on Rust and Python
														for key in tokenizer_pp.special_tokens_map:
																		self.assertTrue(hasattr(A__       ,A__		)		)

														shutil.rmtree(A__		)

														# Save tokenizer rust, legacy_format=False
														_A	:				int   =      tempfile.mkdtemp()

														_A	:				List[Any]   =      tokenizer_r.save_pretrained(A__       ,legacy_format=A__		)
														_A	:				Dict   =      tokenizer_p.save_pretrained(A__		)

														# Checks it saved the tokenizer.json file
														self.assertTrue(any('''tokenizer.json''' in f for f in tokenizer_r_files		)		)

														# Checks everything loads correctly in the same way
														_A	:				Optional[int]   =      tokenizer_r.from_pretrained(A__		)
														_A	:				str   =      tokenizer_p.from_pretrained(A__		)

														# Check special tokens are set accordingly on Rust and Python
														for key in tokenizer_pp.special_tokens_map:
																		self.assertTrue(hasattr(A__       ,A__		)		)

														shutil.rmtree(A__		)







		@require_torch
		def  A__      (					self		):
						if not self.test_seqaseq:
										return

						_A	:				Union[str, Any]   =      self.get_tokenizers()
						for tokenizer in tokenizers:
										with self.subTest(f"""{tokenizer.__class__.__name__}"""		):
														# Longer text that will definitely require truncation.
														_A	:				Union[str, Any]   =      [
														    ''' UN Chief Says There Is No Military Solution in Syria''',
														    ''' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for'''
														    ''' Syria is that \'there is no military solution\' to the nearly five-year conflict and more weapons'''
														    ''' will only worsen the violence and misery for millions of people.''',
														]
														_A	:				Tuple   =      [
														    '''Şeful ONU declară că nu există o soluţie militară în Siria''',
														    '''Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al'''
														    ''' Rusiei pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi'''
														    ''' că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.''',
														]
														try:
																		_A	:				List[Any]   =      tokenizer.prepare_seqaseq_batch(
																		    src_texts=A__       ,tgt_texts=A__       ,max_length=3       ,max_target_length=10       ,return_tensors='''pt'''       ,src_lang='''eng_Latn'''       ,tgt_lang='''ron_Latn'''       ,)
														except NotImplementedError:
																		return
														self.assertEqual(batch.input_ids.shape[1]       ,3		)
														self.assertEqual(batch.labels.shape[1]       ,10		)
														# max_target_length will default to max_length if not specified
														_A	:				int   =      tokenizer.prepare_seqaseq_batch(
														    A__       ,tgt_texts=A__       ,max_length=3       ,return_tensors='''pt'''		)
														self.assertEqual(batch.input_ids.shape[1]       ,3		)
														self.assertEqual(batch.labels.shape[1]       ,3		)

														_A	:				Union[str, Any]   =      tokenizer.prepare_seqaseq_batch(
														    src_texts=A__       ,max_length=3       ,max_target_length=10       ,return_tensors='''pt'''		)
														self.assertEqual(batch_encoder_only.input_ids.shape[1]       ,3		)
														self.assertEqual(batch_encoder_only.attention_mask.shape[1]       ,3		)
														self.assertNotIn('''decoder_input_ids'''       ,A__		)







		@unittest.skip('''Unfortunately way too slow to build a BPE with SentencePiece.'''		)
		def  A__      (					self		):
						pass







		def  A__      (					self		):
						for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
										with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""		):
														_A	:				List[str]   =      [AddedToken('''<special>'''       ,lstrip=A__		)]

														_A	:				List[Any]   =      self.rust_tokenizer_class.from_pretrained(
														    A__       ,additional_special_tokens=A__       ,**A__		)
														_A	:				List[str]   =      tokenizer_r.encode('''Hey this is a <special> token'''		)

														_A	:				List[Any]   =      tokenizer_r.encode('''<special>'''       ,add_special_tokens=A__		)[0]

														self.assertTrue(special_token_id in r_output		)

														if self.test_slow_tokenizer:
																		_A	:				Dict   =      self.rust_tokenizer_class.from_pretrained(
																		    A__       ,additional_special_tokens=A__       ,**A__       ,)
																		_A	:				List[Any]   =      self.tokenizer_class.from_pretrained(
																		    A__       ,additional_special_tokens=A__       ,**A__		)

																		_A	:				int   =      tokenizer_p.encode('''Hey this is a <special> token'''		)

																		_A	:				Optional[int]   =      tokenizer_cr.encode('''Hey this is a <special> token'''		)

																		self.assertEqual(A__       ,A__		)
																		self.assertEqual(A__       ,A__		)
																		self.assertTrue(special_token_id in p_output		)
																		self.assertTrue(special_token_id in cr_output		)






@require_torch
@require_sentencepiece
@require_tokenizers
class 						UpperCAmelCase__ (							unittest.TestCase							):
		__snake_case      :       Union[str, Any]     =   "facebook/nllb-200-distilled-600M"
		__snake_case      :       List[Any]     =   [
		    " UN Chief Says There Is No Military Solution in Syria",
		    " Secretary-General Ban Ki-moon says his response to Russia's stepped up military support for Syria is that \"there is no military solution\" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.",
		]
		__snake_case      :       List[str]     =   [
		    "Şeful ONU declară că nu există o soluţie militară în Siria",
		    "Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei"
		    " pentru Siria este că \"nu există o soluţie militară\" la conflictul de aproape cinci ani şi că noi arme nu vor"
		    " face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.",
		]
		__snake_case      :       int     =   [
		    2_5_6_0_4_7,
		    1_6_2_9_7,
		    1_3_4_4_0_8,
		    8_1_6_5,
		    2_4_8_0_6_6,
		    1_4_7_3_4,
		    9_5_0,
		    1_1_3_5,
		    1_0_5_7_2_1,
		    3_5_7_3,
		    8_3,
		    2_7_3_5_2,
		    1_0_8,
		    4_9_4_8_6,
		    2,
		]







		@classmethod
		def  A__      (					cls		):
						_A	:				NllbTokenizer   =      NllbTokenizer.from_pretrained(
						    cls.checkpoint_name       ,src_lang='''eng_Latn'''       ,tgt_lang='''ron_Latn'''		)
						_A	:				List[str]   =      1
						return cls







		def  A__      (					self		):
						self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Arab''']       ,256001		)
						self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''ace_Latn''']       ,256002		)
						self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['''fra_Latn''']       ,256057		)







		def  A__      (					self		):
						_A	:				Dict   =      self.tokenizer.batch_encode_plus(self.src_text		).input_ids[0]
						self.assertListEqual(self.expected_src_tokens       ,A__		)







		def  A__      (					self		):
						self.assertIn(A__       ,self.tokenizer.all_special_ids		)
						# fmt: off
						_A	:				Dict   =      [RO_CODE, 4254, 98068, 112923, 39072, 3909, 713, 102767, 26, 17314, 35642, 14683, 33118, 2022, 66987, 2, 256047]
						# fmt: on

						_A	:				Any   =      self.tokenizer.decode(A__       ,skip_special_tokens=A__		)
						_A	:				Any   =      self.tokenizer.decode(generated_ids[1:]       ,skip_special_tokens=A__		)
						self.assertEqual(A__       ,A__		)
						self.assertNotIn(self.tokenizer.eos_token       ,A__		)







		def  A__      (					self		):
						_A	:				int   =      ['''this is gunna be a long sentence ''' * 20]
						assert isinstance(src_text[0]       ,A__		)
						_A	:				Tuple   =      10
						_A	:				Optional[Any]   =      self.tokenizer(A__       ,max_length=A__       ,truncation=A__		).input_ids[0]
						self.assertEqual(ids[-1]       ,2		)
						self.assertEqual(ids[0]       ,A__		)
						self.assertEqual(len(A__		)       ,A__		)







		def  A__      (					self		):
						self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['''<mask>''', '''ar_AR''']		)       ,[256203, 3]		)







		def  A__      (					self		):
						_A	:				str   =      tempfile.mkdtemp()
						_A	:				str   =      self.tokenizer.fairseq_tokens_to_ids
						self.tokenizer.save_pretrained(A__		)
						_A	:				Any   =      NllbTokenizer.from_pretrained(A__		)
						self.assertDictEqual(new_tok.fairseq_tokens_to_ids       ,A__		)







		@require_torch
		def  A__      (					self		):
						_A	:				Optional[int]   =      self.tokenizer(
						    self.src_text       ,text_target=self.tgt_text       ,padding=A__       ,truncation=A__       ,max_length=len(self.expected_src_tokens		)       ,return_tensors='''pt'''       ,)
						_A	:				str   =      shift_tokens_right(
						    batch['''labels''']       ,self.tokenizer.pad_token_id       ,self.tokenizer.lang_code_to_id['''ron_Latn''']		)

						self.assertIsInstance(A__       ,A__		)

						self.assertEqual((2, 15)       ,batch.input_ids.shape		)
						self.assertEqual((2, 15)       ,batch.attention_mask.shape		)
						_A	:				Dict   =      batch.input_ids.tolist()[0]
						self.assertListEqual(self.expected_src_tokens       ,A__		)
						self.assertEqual(A__       ,batch.decoder_input_ids[0, 0]		)  # EOS
						# Test that special tokens are reset
						self.assertEqual(self.tokenizer.prefix_tokens       ,[EN_CODE]		)
						self.assertEqual(self.tokenizer.suffix_tokens       ,[self.tokenizer.eos_token_id]		)







		def  A__      (					self		):
						_A	:				str   =      self.tokenizer(self.src_text       ,padding=A__       ,truncation=A__       ,max_length=3       ,return_tensors='''pt'''		)
						_A	:				Tuple   =      self.tokenizer(
						    text_target=self.tgt_text       ,padding=A__       ,truncation=A__       ,max_length=10       ,return_tensors='''pt'''		)
						_A	:				int   =      targets['''input_ids''']
						_A	:				Dict   =      shift_tokens_right(
						    A__       ,self.tokenizer.pad_token_id       ,decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]       ,)

						self.assertEqual(batch.input_ids.shape[1]       ,3		)
						self.assertEqual(batch.decoder_input_ids.shape[1]       ,10		)







		@require_torch
		def  A__      (					self		):
						_A	:				str   =      self.tokenizer._build_translation_inputs(
						    '''A test'''       ,return_tensors='''pt'''       ,src_lang='''eng_Latn'''       ,tgt_lang='''fra_Latn'''		)

						self.assertEqual(
						    nested_simplify(A__		)       ,{
						        # A, test, EOS, en_XX
						        '''input_ids''': [[256047, 70, 7356, 2]],
						        '''attention_mask''': [[1, 1, 1, 1]],
						        # ar_AR
						        '''forced_bos_token_id''': 256057,
						    }       ,)







		@require_torch
		def  A__      (					self		):
						_A	:				int   =      True
						_A	:				Optional[Any]   =      self.tokenizer(
						    '''UN Chief says there is no military solution in Syria'''       ,src_lang='''eng_Latn'''       ,tgt_lang='''fra_Latn'''		)
						self.assertEqual(
						    inputs.input_ids       ,[16297, 134408, 25653, 6370, 248, 254, 103929, 94995, 108, 49486, 2, 256047]		)

						_A	:				List[str]   =      False
						_A	:				Union[str, Any]   =      self.tokenizer(
						    '''UN Chief says there is no military solution in Syria'''       ,src_lang='''eng_Latn'''       ,tgt_lang='''fra_Latn'''		)
						self.assertEqual(
						    inputs.input_ids       ,[256047, 16297, 134408, 25653, 6370, 248, 254, 103929, 94995, 108, 49486, 2]		)



