





import tempfile
import unittest

from transformers import SPIECE_UNDERLINE, BatchEncoding, PLBartTokenizer, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


A       :   List[Any]					= get_tests_dir('''fixtures/test_sentencepiece.model''')


if is_torch_available():
					from transformers.models.plbart.modeling_plbart import shift_tokens_right

A       :   int					= 5_0_0_0_3
A       :   Dict					= 5_0_0_0_2


@require_sentencepiece
@require_tokenizers
class     A     (SCREAMING_SNAKE_CASE  ,	unittest.TestCase       ):
			'''simple docstring'''




			__lowerCamelCase      :			str							     =       PLBartTokenizer
			__lowerCamelCase      :			Optional[Any]							     =       None
			__lowerCamelCase      :			Optional[Any]							     =       False






			def   a_			(     self  :       Tuple  )      ->      str:







					"""simple docstring"""






					super().setUp()

					# We have a SentencePiece fixture for testing
					A__     =  PLBartTokenizer(__lowerCAmelCase				,      language_codes="""base"""				,      keep_accents=__lowerCAmelCase  )
					tokenizer.save_pretrained(self.tmpdirname  )






			def   a_			(     self  :       Optional[Any]  )      ->      List[Any]:







					"""simple docstring"""






					A__     =  PLBartTokenizer(__lowerCAmelCase				,      language_codes="""base"""				,      keep_accents=__lowerCAmelCase  )

					A__     =  tokenizer.tokenize("""This is a test"""  )
					self.assertListEqual(__lowerCAmelCase				,      ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]  )

					self.assertListEqual(
					    tokenizer.convert_tokens_to_ids(__lowerCAmelCase  )				,      [value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]				,      )

					A__     =  tokenizer.tokenize("""I was born in 92000, and this is falsé."""  )
					self.assertListEqual(
					    __lowerCAmelCase				,      [
					        SPIECE_UNDERLINE + """I""",
					        SPIECE_UNDERLINE + """was""",
					        SPIECE_UNDERLINE + """b""",
					        """or""",
					        """n""",
					        SPIECE_UNDERLINE + """in""",
					        SPIECE_UNDERLINE + """""",
					        """9""",
					        """2""",
					        """0""",
					        """0""",
					        """0""",
					        """,""",
					        SPIECE_UNDERLINE + """and""",
					        SPIECE_UNDERLINE + """this""",
					        SPIECE_UNDERLINE + """is""",
					        SPIECE_UNDERLINE + """f""",
					        """al""",
					        """s""",
					        """é""",
					        """.""",
					    ]				,      )
					A__     =  tokenizer.convert_tokens_to_ids(__lowerCAmelCase  )
					self.assertListEqual(
					    __lowerCAmelCase				,      [
					        value + tokenizer.fairseq_offset
					        for value in [8, 21, 84, 55, 24, 19, 7, 2, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 2, 4]
					    ]				,      )

					A__     =  tokenizer.convert_ids_to_tokens(__lowerCAmelCase  )
					self.assertListEqual(
					    __lowerCAmelCase				,      [
					        SPIECE_UNDERLINE + """I""",
					        SPIECE_UNDERLINE + """was""",
					        SPIECE_UNDERLINE + """b""",
					        """or""",
					        """n""",
					        SPIECE_UNDERLINE + """in""",
					        SPIECE_UNDERLINE + """""",
					        """<unk>""",
					        """2""",
					        """0""",
					        """0""",
					        """0""",
					        """,""",
					        SPIECE_UNDERLINE + """and""",
					        SPIECE_UNDERLINE + """this""",
					        SPIECE_UNDERLINE + """is""",
					        SPIECE_UNDERLINE + """f""",
					        """al""",
					        """s""",
					        """<unk>""",
					        """.""",
					    ]				,      )

					A__     =  tokenizer.vocab_size
					A__     =  [tokenizer.convert_ids_to_tokens(__lowerCAmelCase  ) for x in range(end - 4				,      __lowerCAmelCase  )]

					self.assertListEqual(__lowerCAmelCase				,      ["""__java__""", """__python__""", """__en_XX__""", """<mask>"""]  )

					A__     =  """java.lang.Exception, python.lang.Exception, javascript, php, ruby, go"""
					A__     =  tokenizer(__lowerCAmelCase  ).input_ids
					self.assertEqual(
					    tokenizer.decode(__lowerCAmelCase				,      skip_special_tokens=__lowerCAmelCase				,      clean_up_tokenization_spaces=__lowerCAmelCase  )				,      __lowerCAmelCase				,      )







			def   a_			(     self  :       int  )      ->      Union[str, Any]:







					"""simple docstring"""






					A__     =  PLBartTokenizer(__lowerCAmelCase				,      language_codes="""multi"""				,      keep_accents=__lowerCAmelCase  )

					A__     =  tokenizer.tokenize("""This is a test"""  )
					self.assertListEqual(__lowerCAmelCase				,      ["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]  )

					self.assertListEqual(
					    tokenizer.convert_tokens_to_ids(__lowerCAmelCase  )				,      [value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]				,      )

					A__     =  tokenizer.tokenize("""I was born in 92000, and this is falsé."""  )
					self.assertListEqual(
					    __lowerCAmelCase				,      [
					        SPIECE_UNDERLINE + """I""",
					        SPIECE_UNDERLINE + """was""",
					        SPIECE_UNDERLINE + """b""",
					        """or""",
					        """n""",
					        SPIECE_UNDERLINE + """in""",
					        SPIECE_UNDERLINE + """""",
					        """9""",
					        """2""",
					        """0""",
					        """0""",
					        """0""",
					        """,""",
					        SPIECE_UNDERLINE + """and""",
					        SPIECE_UNDERLINE + """this""",
					        SPIECE_UNDERLINE + """is""",
					        SPIECE_UNDERLINE + """f""",
					        """al""",
					        """s""",
					        """é""",
					        """.""",
					    ]				,      )
					A__     =  tokenizer.convert_tokens_to_ids(__lowerCAmelCase  )
					self.assertListEqual(
					    __lowerCAmelCase				,      [
					        value + tokenizer.fairseq_offset
					        for value in [8, 21, 84, 55, 24, 19, 7, 2, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 2, 4]
					    ]				,      )

					A__     =  tokenizer.convert_ids_to_tokens(__lowerCAmelCase  )
					self.assertListEqual(
					    __lowerCAmelCase				,      [
					        SPIECE_UNDERLINE + """I""",
					        SPIECE_UNDERLINE + """was""",
					        SPIECE_UNDERLINE + """b""",
					        """or""",
					        """n""",
					        SPIECE_UNDERLINE + """in""",
					        SPIECE_UNDERLINE + """""",
					        """<unk>""",
					        """2""",
					        """0""",
					        """0""",
					        """0""",
					        """,""",
					        SPIECE_UNDERLINE + """and""",
					        SPIECE_UNDERLINE + """this""",
					        SPIECE_UNDERLINE + """is""",
					        SPIECE_UNDERLINE + """f""",
					        """al""",
					        """s""",
					        """<unk>""",
					        """.""",
					    ]				,      )
					A__     =  tokenizer.vocab_size
					A__     =  [tokenizer.convert_ids_to_tokens(__lowerCAmelCase  ) for x in range(end - 7				,      __lowerCAmelCase  )]

					self.assertListEqual(
					    __lowerCAmelCase				,      ["""__java__""", """__python__""", """__en_XX__""", """__javascript__""", """__php__""", """__ruby__""", """__go__"""]  )
					A__     =  """java.lang.Exception, python.lang.Exception, javascript, php, ruby, go"""
					A__     =  tokenizer(__lowerCAmelCase  ).input_ids
					self.assertEqual(
					    tokenizer.decode(__lowerCAmelCase				,      skip_special_tokens=__lowerCAmelCase				,      clean_up_tokenization_spaces=__lowerCAmelCase  )				,      __lowerCAmelCase				,      )


@require_torch
@require_sentencepiece
@require_tokenizers
class     A     (unittest.TestCase       ):
			'''simple docstring'''




			__lowerCamelCase      :			Tuple							     =       '''uclanlp/plbart-python-en_XX'''
			__lowerCamelCase      :			str							     =       [
			    '''def maximum(a,b,c):NEW_LINE_INDENTreturn max([a,b,c])''',
			    '''def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])''',
			]
			__lowerCamelCase      :			Any							     =       [
			    '''Returns the maximum value of a b c.''',
			    '''Sums the values of a b c.''',
			]
			__lowerCamelCase      :			Optional[Any]							     =       [
			    134,
			    5452,
			    33460,
			    33441,
			    33463,
			    33465,
			    33463,
			    33449,
			    988,
			    20,
			    33456,
			    19,
			    33456,
			    771,
			    39,
			    4258,
			    889,
			    3318,
			    33441,
			    33463,
			    33465,
			    33463,
			    33449,
			    2471,
			    2,
			    PYTHON_CODE,
			]






			@classmethod
			def   a_			(     cls  :       List[Any]  )      ->      Optional[Any]:







					"""simple docstring"""






					A__     =  PLBartTokenizer.from_pretrained(
					    cls.checkpoint_name				,      language_codes="""base"""				,      src_lang="""python"""				,      tgt_lang="""en_XX"""  )
					A__     =  1
					return cls






			def   a_			(     self  :       str  )      ->      int:







					"""simple docstring"""






					self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""__java__"""]				,      5_00_01  )
					self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""__python__"""]				,      5_00_02  )
					self.assertEqual(self.tokenizer.fairseq_tokens_to_ids["""__en_XX__"""]				,      5_00_03  )






			def   a_			(     self  :       Optional[Any]  )      ->      List[str]:







					"""simple docstring"""






					A__     =  self.tokenizer.batch_encode_plus(self.src_text  ).input_ids[0]
					self.assertListEqual(self.expected_src_tokens				,      __lowerCAmelCase  )






			def   a_			(     self  :       Optional[int]  )      ->      Dict:







					"""simple docstring"""






					self.assertIn(__lowerCAmelCase				,      self.tokenizer.all_special_ids  )
					A__     =  [EN_CODE, 90_37, 3_34_42, 57, 7_52, 1_53, 14, 56, 18, 9, 2]
					A__     =  self.tokenizer.decode(__lowerCAmelCase				,      skip_special_tokens=__lowerCAmelCase  )
					A__     =  self.tokenizer.decode(generated_ids[1:]				,      skip_special_tokens=__lowerCAmelCase  )
					self.assertEqual(__lowerCAmelCase				,      __lowerCAmelCase  )
					self.assertNotIn(self.tokenizer.eos_token				,      __lowerCAmelCase  )






			def   a_			(     self  :       Union[str, Any]  )      ->      int:







					"""simple docstring"""






					A__     =  ["""def sum(a,b,c):NEW_LINE_INDENTreturn sum([a,b,c])""" * 20]
					self.assertIsInstance(src_text[0]				,      __lowerCAmelCase  )
					A__     =  10
					A__     =  self.tokenizer(__lowerCAmelCase				,      max_length=__lowerCAmelCase				,      truncation=__lowerCAmelCase  ).input_ids[0]
					self.assertEqual(ids[-2]				,      2  )
					self.assertEqual(ids[-1]				,      __lowerCAmelCase  )
					self.assertEqual(len(__lowerCAmelCase  )				,      __lowerCAmelCase  )






			def   a_			(     self  :       List[str]  )      ->      int:







					"""simple docstring"""






					self.assertListEqual(self.tokenizer.convert_tokens_to_ids(["""<mask>""", """__java__"""]  )				,      [5_00_04, 5_00_01]  )






			def   a_			(     self  :       int  )      ->      Dict:







					"""simple docstring"""






					A__     =  tempfile.mkdtemp()
					A__     =  self.tokenizer.fairseq_tokens_to_ids
					self.tokenizer.save_pretrained(__lowerCAmelCase  )
					A__     =  PLBartTokenizer.from_pretrained(__lowerCAmelCase  )
					self.assertDictEqual(new_tok.fairseq_tokens_to_ids				,      __lowerCAmelCase  )






			@require_torch
			def   a_			(     self  :       Optional[Any]  )      ->      Optional[Any]:







					"""simple docstring"""






					A__     =  self.tokenizer(self.src_text				,      text_target=self.tgt_text				,      padding=__lowerCAmelCase				,      return_tensors="""pt"""  )
					A__     =  shift_tokens_right(batch["""labels"""]				,      self.tokenizer.pad_token_id  )

					# fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
					self.assertEqual(batch.input_ids[1][-2:].tolist()				,      [2, PYTHON_CODE]  )
					self.assertEqual(batch.decoder_input_ids[1][0]				,      __lowerCAmelCase  )
					self.assertEqual(batch.decoder_input_ids[1][-1]				,      2  )
					self.assertEqual(batch.labels[1][-2:].tolist()				,      [2, EN_CODE]  )






			@require_torch
			def   a_			(     self  :       Any  )      ->      Optional[Any]:







					"""simple docstring"""






					A__     =  self.tokenizer(
					    self.src_text				,      text_target=self.tgt_text				,      padding=__lowerCAmelCase				,      truncation=__lowerCAmelCase				,      max_length=len(self.expected_src_tokens  )				,      return_tensors="""pt"""				,      )
					A__     =  shift_tokens_right(batch["""labels"""]				,      self.tokenizer.pad_token_id  )

					self.assertIsInstance(__lowerCAmelCase				,      __lowerCAmelCase  )

					self.assertEqual((2, 26)				,      batch.input_ids.shape  )
					self.assertEqual((2, 26)				,      batch.attention_mask.shape  )
					A__     =  batch.input_ids.tolist()[0]
					self.assertListEqual(self.expected_src_tokens				,      __lowerCAmelCase  )
					self.assertEqual(2				,      batch.decoder_input_ids[0, -1]  )  # EOS
					# Test that special tokens are reset
					self.assertEqual(self.tokenizer.prefix_tokens				,      []  )
					self.assertEqual(self.tokenizer.suffix_tokens				,      [self.tokenizer.eos_token_id, PYTHON_CODE]  )






			def   a_			(     self  :       Dict  )      ->      List[Any]:







					"""simple docstring"""






					A__     =  self.tokenizer(self.src_text				,      padding=__lowerCAmelCase				,      truncation=__lowerCAmelCase				,      max_length=3				,      return_tensors="""pt"""  )
					A__     =  self.tokenizer(
					    text_target=self.tgt_text				,      padding=__lowerCAmelCase				,      truncation=__lowerCAmelCase				,      max_length=10				,      return_tensors="""pt"""  )
					A__     =  targets["""input_ids"""]
					A__     =  shift_tokens_right(__lowerCAmelCase				,      self.tokenizer.pad_token_id  )

					self.assertEqual(batch.input_ids.shape[1]				,      3  )
					self.assertEqual(batch.decoder_input_ids.shape[1]				,      10  )







			@require_torch
			def   a_			(     self  :       List[Any]  )      ->      List[Any]:







					"""simple docstring"""






					A__     =  self.tokenizer._build_translation_inputs(
					    """A test"""				,      return_tensors="""pt"""				,      src_lang="""en_XX"""				,      tgt_lang="""java"""  )

					self.assertEqual(
					    nested_simplify(__lowerCAmelCase  )				,      {
					        # A, test, EOS, en_XX
					        """input_ids""": [[1_50, 2_42, 2, 5_00_03]],
					        """attention_mask""": [[1, 1, 1, 1]],
					        # java
					        """forced_bos_token_id""": 5_00_01,
					    }				,      )










import warnings
from contextlib import contextmanager

from ...processing_utils import ProcessorMixin
from .feature_extraction_wavaveca import WavaVecaFeatureExtractor
from .tokenization_wavaveca import WavaVecaCTCTokenizer


class     A     (SCREAMING_SNAKE_CASE       ):
			'''simple docstring'''




			__lowerCamelCase      :			Dict							     =       '''Wav2Vec2FeatureExtractor'''
			__lowerCamelCase      :			str							     =       '''AutoTokenizer'''






			def __init__(     self  :       Union[str, Any]				,      __lowerCAmelCase  :       List[Any]				,      __lowerCAmelCase  :       Optional[Any]  )      ->      Tuple:







					"""simple docstring"""






					super().__init__(__lowerCAmelCase				,      __lowerCAmelCase  )
					A__     =  self.feature_extractor
					A__     =  False






			@classmethod
			def   a_			(     cls  :       Union[str, Any]				,      __lowerCAmelCase  :       Optional[Any]				,      **__lowerCAmelCase  :       Union[str, Any]  )      ->      int:







					"""simple docstring"""






					try:
							return super().from_pretrained(__lowerCAmelCase				,      **__lowerCAmelCase  )
					except OSError:
							warnings.warn(
							    f'Loading a tokenizer inside {cls.__name__} from a config that does not'
							    """ include a `tokenizer_class` attribute is deprecated and will be """
							    """removed in v5. Please add `'tokenizer_class': 'Wav2Vec2CTCTokenizer'`"""
							    """ attribute to either your `config.json` or `tokenizer_config.json` """
							    """file to suppress this warning: """				,      __lowerCAmelCase				,      )

							A__     =  WavaVecaFeatureExtractor.from_pretrained(__lowerCAmelCase				,      **__lowerCAmelCase  )
							A__     =  WavaVecaCTCTokenizer.from_pretrained(__lowerCAmelCase				,      **__lowerCAmelCase  )

							return cls(feature_extractor=__lowerCAmelCase				,      tokenizer=__lowerCAmelCase  )






			def __call__(     self  :       Any				,      *__lowerCAmelCase  :       Union[str, Any]				,      **__lowerCAmelCase  :       List[Any]  )      ->      Optional[Any]:







					"""simple docstring"""






					if self._in_target_context_manager:
							return self.current_processor(*__lowerCAmelCase				,      **__lowerCAmelCase  )

					if "raw_speech" in kwargs:
							warnings.warn("""Using `raw_speech` as a keyword argument is deprecated. Use `audio` instead."""  )
							A__     =  kwargs.pop("""raw_speech"""  )
					else:
							A__     =  kwargs.pop("""audio"""				,      __lowerCAmelCase  )
					A__     =  kwargs.pop("""sampling_rate"""				,      __lowerCAmelCase  )
					A__     =  kwargs.pop("""text"""				,      __lowerCAmelCase  )
					if len(__lowerCAmelCase  ) > 0:
							A__     =  args[0]
							A__     =  args[1:]

					if audio is None and text is None:
							raise ValueError("""You need to specify either an `audio` or `text` input to process."""  )

					if audio is not None:
							A__     =  self.feature_extractor(__lowerCAmelCase				,      *__lowerCAmelCase				,      sampling_rate=__lowerCAmelCase				,      **__lowerCAmelCase  )
					if text is not None:
							A__     =  self.tokenizer(__lowerCAmelCase				,      **__lowerCAmelCase  )

					if text is None:
							return inputs
					elif audio is None:
							return encodings
					else:
							A__     =  encodings["""input_ids"""]
							return inputs






			def   a_			(     self  :       List[str]				,      *__lowerCAmelCase  :       Optional[Any]				,      **__lowerCAmelCase  :       Tuple  )      ->      List[str]:







					"""simple docstring"""






					if self._in_target_context_manager:
							return self.current_processor.pad(*__lowerCAmelCase				,      **__lowerCAmelCase  )

					A__     =  kwargs.pop("""input_features"""				,      __lowerCAmelCase  )
					A__     =  kwargs.pop("""labels"""				,      __lowerCAmelCase  )
					if len(__lowerCAmelCase  ) > 0:
							A__     =  args[0]
							A__     =  args[1:]

					if input_features is not None:
							A__     =  self.feature_extractor.pad(__lowerCAmelCase				,      *__lowerCAmelCase				,      **__lowerCAmelCase  )
					if labels is not None:
							A__     =  self.tokenizer.pad(__lowerCAmelCase				,      **__lowerCAmelCase  )

					if labels is None:
							return input_features
					elif input_features is None:
							return labels
					else:
							A__     =  labels["""input_ids"""]
							return input_features






			def   a_			(     self  :       Optional[int]				,      *__lowerCAmelCase  :       List[str]				,      **__lowerCAmelCase  :       List[Any]  )      ->      Dict:







					"""simple docstring"""






					return self.tokenizer.batch_decode(*__lowerCAmelCase				,      **__lowerCAmelCase  )






			def   a_			(     self  :       Any				,      *__lowerCAmelCase  :       List[str]				,      **__lowerCAmelCase  :       Union[str, Any]  )      ->      List[str]:







					"""simple docstring"""






					return self.tokenizer.decode(*__lowerCAmelCase				,      **__lowerCAmelCase  )







			@contextmanager
			def   a_			(     self  :       Union[str, Any]  )      ->      int:







					"""simple docstring"""






					warnings.warn(
					    """`as_target_processor` is deprecated and will be removed in v5 of Transformers. You can process your """
					    """labels by using the argument `text` of the regular `__call__` method (either in the same call as """
					    """your audio inputs, or in a separate call."""  )
					A__     =  True
					A__     =  self.tokenizer
					yield
					A__     =  self.feature_extractor
					A__     =  False



