




'''simple docstring'''







import tempfile
import unittest
from pathlib import Path
from shutil import copyfile

from transformers import MaMaaaTokenizer, is_torch_available
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
    slow,
)
from transformers.utils import is_sentencepiece_available


if is_sentencepiece_available():
      from transformers.models.mam_aaa.tokenization_mam_aaa import VOCAB_FILES_NAMES, save_json

from ...test_tokenization_common import TokenizerTesterMixin


if is_sentencepiece_available():
      lowercase      :   List[str]      =       get_tests_dir("""fixtures/test_sentencepiece.model""")


if is_torch_available():
      from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

lowercase      :   Union[str, Any]      =       12_8022
lowercase      :   Dict      =       12_8028







@require_sentencepiece
class 					_a (a__,		unittest.TestCase	):





     '''simple docstring'''



     lowerCAmelCase_  :     Tuple          =					MaMaaaTokenizer
     lowerCAmelCase_  :     List[Any]          =					False
     lowerCAmelCase_  :     List[Any]          =					False
     lowerCAmelCase_  :     int          =					True







     def 			snake_case_      (				self							)      ->				str:
          super().setUp()

          snake_case			:      Dict        =			["""</s>""", """<unk>""", """▁This""", """▁is""", """▁a""", """▁t""", """est""", """\u0120""", """<pad>"""]
          snake_case			:      int        =			dict(zip(__a							,range(len(__a							)							)							)							)
          snake_case			:      List[Any]        =			Path(self.tmpdirname							)
          save_json(__a							,save_dir / VOCAB_FILES_NAMES["""vocab_file"""]							)
          if not (save_dir / VOCAB_FILES_NAMES["spm_file"]).exists():
               copyfile(__a							,save_dir / VOCAB_FILES_NAMES["""spm_file"""]							)

          snake_case			:      List[str]        =			MaMaaaTokenizer.from_pretrained(self.tmpdirname							)
          tokenizer.save_pretrained(self.tmpdirname							)







     def 			snake_case_      (				self							,**__a							)      ->				Tuple:
          return MaMaaaTokenizer.from_pretrained(self.tmpdirname							,**__a							)







     def 			snake_case_      (				self							,__a							)      ->				List[Any]:
          return (
              "This is a test",
              "This is a test",
          )







     def 			snake_case_      (				self							)      ->				Tuple:
          snake_case			:      Any        =			"""</s>"""
          snake_case			:      List[Any]        =			0

          self.assertEqual(self.get_tokenizer()._convert_token_to_id(__a							)							,__a							)
          self.assertEqual(self.get_tokenizer()._convert_id_to_token(__a							)							,__a							)







     def 			snake_case_      (				self							)      ->				Optional[Any]:
          snake_case			:      Tuple        =			self.get_tokenizer()
          snake_case			:      Union[str, Any]        =			list(tokenizer.get_vocab().keys()							)

          self.assertEqual(vocab_keys[0]							,"""</s>"""							)
          self.assertEqual(vocab_keys[1]							,"""<unk>"""							)
          self.assertEqual(vocab_keys[-1]							,"""<s>"""							)
          self.assertEqual(len(__a							)							,tokenizer.vocab_size + len(tokenizer.get_added_vocab()							)							)







     @unittest.skip("""Skip this test while all models are still to be uploaded."""							)
     def 			snake_case_      (				self							)      ->				List[str]:
          pass







     def 			snake_case_      (				self							)      ->				Optional[Any]:
          snake_case			:      List[str]        =			self.get_tokenizer()

          snake_case			:      Union[str, Any]        =			tokenizer.tokenize("""This is a test"""							)
          self.assertListEqual(__a							,["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]							)

          self.assertListEqual(
              tokenizer.convert_tokens_to_ids(__a							)							,[2, 3, 4, 5, 6]							,)

          snake_case			:      List[str]        =			tokenizer.convert_ids_to_tokens([2, 3, 4, 5, 6]							)
          self.assertListEqual(__a							,["""▁This""", """▁is""", """▁a""", """▁t""", """est"""]							)

          snake_case			:      str        =			tokenizer.convert_tokens_to_string(__a							)
          self.assertEqual(__a							,"""This is a test"""							)







     @slow
     def 			snake_case_      (				self							)      ->				str:
          # fmt: off
          snake_case			:      Optional[int]        =			{"""input_ids""": [[128_022, 110_108, 397, 11, 38_272, 2_247, 124_811, 285, 18_105, 1_586, 207, 7, 39_534, 4_428, 397, 1_019, 18_105, 1_586, 207, 7, 41_337, 16_786, 241, 7, 20_214, 17, 125_690, 10_398, 7, 44_378, 58_069, 68_342, 7_798, 7_343, 11, 299, 33_310, 4, 158, 37_350, 94_077, 4_569, 299, 33_310, 90, 4, 52_840, 290, 4, 31_270, 112, 299, 682, 4, 52_840, 39_953, 14_079, 193, 52_519, 90_894, 17_894, 120_697, 11, 40_445, 551, 17, 1_019, 52_519, 90_894, 17_756, 963, 11, 40_445, 480, 17, 9_792, 1_120, 5_173, 1_393, 6_240, 16_786, 241, 120_996, 28, 1_245, 1_393, 118_240, 11_123, 1_019, 93_612, 2_691, 10_618, 98_058, 120_409, 1_928, 279, 4, 40_683, 367, 178, 207, 1_019, 103, 103_121, 506, 65_296, 5, 2], [128_022, 21_217, 367, 117, 125_450, 128, 719, 7, 7_308, 40, 93_612, 12_669, 1_116, 16_704, 71, 17_785, 3_699, 15_592, 35, 144, 9_584, 241, 11_943, 713, 950, 799, 2_247, 88_427, 150, 149, 118_813, 120_706, 1_019, 106_906, 81_518, 28, 1_224, 22_799, 397, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [128_022, 1_658, 123_311, 5_155, 5_578, 4_722, 279, 14_947, 2_366, 1_120, 1_197, 14, 1_348, 9_232, 5, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]], """attention_mask""": [[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]]}  # noqa: E501
          # fmt: on

          self.tokenizer_integration_test_util(
              expected_encoding=__a							,model_name="""facebook/m2m100_418M"""							,revision="""c168bae485c864188cf9aa0e4108b0b6934dc91e"""							,)







@require_torch
@require_sentencepiece
@require_tokenizers
class 					_a (unittest.TestCase	):





     '''simple docstring'''



     lowerCAmelCase_  :     Any          =					"""facebook/m2m100_418M"""
     lowerCAmelCase_  :     Any          =					[
         """In my opinion, there are two levels of response from the French government.""",
         """NSA Affair Emphasizes Complete Lack of Debate on Intelligence""",
     ]
     lowerCAmelCase_  :     str          =					[
         """Selon moi, il y a deux niveaux de réponse de la part du gouvernement français.""",
         """L'affaire NSA souligne l'absence totale de débat sur le renseignement""",
     ]

     # fmt: off
     lowerCAmelCase_  :     int          =					[EN_CODE, 593, 1949, 115781, 4, 71586, 4234, 60633, 126233, 432, 123808, 15592, 1197, 117132, 120618, 5, 2]







     @classmethod
     def 			snake_case_      (				cls							)      ->				List[Any]:
          snake_case			:      MaMaaaTokenizer        =			MaMaaaTokenizer.from_pretrained(
              cls.checkpoint_name							,src_lang="""en"""							,tgt_lang="""fr"""							)
          snake_case			:      str        =			1
          return cls







     def 			snake_case_      (				self							)      ->				Tuple:
          self.assertEqual(self.tokenizer.get_lang_id("""ar"""							)							,128_006							)
          self.assertEqual(self.tokenizer.get_lang_id("""en"""							)							,128_022							)
          self.assertEqual(self.tokenizer.get_lang_id("""ro"""							)							,128_076							)
          self.assertEqual(self.tokenizer.get_lang_id("""mr"""							)							,128_063							)







     def 			snake_case_      (				self							)      ->				int:
          snake_case			:      str        =			self.tokenizer.get_vocab()
          self.assertEqual(len(__a							)							,self.tokenizer.vocab_size							)
          self.assertEqual(vocab["""<unk>"""]							,3							)
          self.assertIn(self.tokenizer.get_lang_token("""en"""							)							,__a							)







     def 			snake_case_      (				self							)      ->				List[str]:
          snake_case			:      str        =			"""en"""
          snake_case			:      Tuple        =			self.tokenizer.batch_encode_plus(self.src_text							).input_ids[0]
          self.assertListEqual(self.expected_src_tokens							,__a							)







     def 			snake_case_      (				self							)      ->				Tuple:
          self.assertIn(__a							,self.tokenizer.all_special_ids							)
          # fmt: off
          snake_case			:      Any        =			[FR_CODE, 5_364, 82, 8_642, 4, 294, 47, 8, 14_028, 136, 3_286, 9_706, 6, 90_797, 6, 144_012, 162, 88_128, 30_061, 5, 2]
          # fmt: on
          snake_case			:      Dict        =			self.tokenizer.decode(__a							,skip_special_tokens=__a							)
          snake_case			:      Optional[int]        =			self.tokenizer.decode(generated_ids[1:]							,skip_special_tokens=__a							)
          self.assertEqual(__a							,__a							)
          self.assertNotIn(self.tokenizer.eos_token							,__a							)







     def 			snake_case_      (				self							)      ->				List[str]:
          snake_case			:      Optional[Any]        =			tempfile.mkdtemp()
          snake_case			:      str        =			self.tokenizer.lang_token_to_id
          self.tokenizer.save_pretrained(__a							)
          snake_case			:      int        =			MaMaaaTokenizer.from_pretrained(__a							)
          self.assertDictEqual(new_tok.lang_token_to_id							,__a							)







     @require_torch
     def 			snake_case_      (				self							)      ->				List[Any]:
          snake_case			:      Any        =			"""en"""
          snake_case			:      Any        =			"""fr"""

          snake_case			:      List[str]        =			self.tokenizer(self.src_text							,text_target=self.tgt_text							,padding=__a							,return_tensors="""pt"""							)

          snake_case			:      List[str]        =			shift_tokens_right(
              batch["""labels"""]							,self.tokenizer.pad_token_id							,self.tokenizer.eos_token_id							)

          for k in batch:
               snake_case			:      Optional[Any]        =			batch[k].tolist()
          # batch = {k: v.tolist() for k,v in batch.items()}
          # fairseq batch: https://gist.github.com/sshleifer/cba08bc2109361a74ac3760a7e30e4f4
          # batch.decoder_inputs_ids[0][0] ==
          assert batch.input_ids[1][0] == EN_CODE
          assert batch.input_ids[1][-1] == 2
          assert batch.labels[1][0] == FR_CODE
          assert batch.labels[1][-1] == 2
          assert batch.decoder_input_ids[1][:2] == [2, FR_CODE]







     @require_torch
     def 			snake_case_      (				self							)      ->				Tuple:
          snake_case			:      Any        =			"""mr"""
          self.assertListEqual(self.tokenizer.prefix_tokens							,[self.tokenizer.get_lang_id("""mr"""							)]							)
          self.assertListEqual(self.tokenizer.suffix_tokens							,[self.tokenizer.eos_token_id]							)

          snake_case			:      List[Any]        =			"""zh"""
          self.assertListEqual(self.tokenizer.prefix_tokens							,[self.tokenizer.get_lang_id("""zh"""							)]							)
          self.assertListEqual(self.tokenizer.suffix_tokens							,[self.tokenizer.eos_token_id]							)







     @require_torch
     def 			snake_case_      (				self							)      ->				Any:
          snake_case			:      List[Any]        =			"""mr"""
          self.tokenizer._switch_to_target_mode()
          self.assertListEqual(self.tokenizer.prefix_tokens							,[self.tokenizer.get_lang_id("""mr"""							)]							)
          self.assertListEqual(self.tokenizer.suffix_tokens							,[self.tokenizer.eos_token_id]							)
          self.tokenizer._switch_to_input_mode()
          self.assertListEqual(self.tokenizer.prefix_tokens							,[self.tokenizer.get_lang_id(self.tokenizer.src_lang							)]							)

          snake_case			:      Tuple        =			"""zh"""
          self.tokenizer._switch_to_target_mode()
          self.assertListEqual(self.tokenizer.prefix_tokens							,[self.tokenizer.get_lang_id("""zh"""							)]							)
          self.assertListEqual(self.tokenizer.suffix_tokens							,[self.tokenizer.eos_token_id]							)
          self.tokenizer._switch_to_input_mode()
          self.assertListEqual(self.tokenizer.prefix_tokens							,[self.tokenizer.get_lang_id(self.tokenizer.src_lang							)]							)







     @require_torch
     def 			snake_case_      (				self							)      ->				Dict:
          snake_case			:      Union[str, Any]        =			self.tokenizer._build_translation_inputs("""A test"""							,return_tensors="""pt"""							,src_lang="""en"""							,tgt_lang="""ar"""							)

          self.assertEqual(
              nested_simplify(__a							)							,{
                  # en_XX, A, test, EOS
                  """input_ids""": [[128_022, 58, 4_183, 2]],
                  """attention_mask""": [[1, 1, 1, 1]],
                  # ar_AR
                  """forced_bos_token_id""": 128_006,
              }							,)









'''simple docstring'''







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from .tokenization_lxmert import LxmertTokenizer


lowercase      :   Tuple      =       {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

lowercase      :   Dict      =       {
    """vocab_file""": {
        """unc-nlp/lxmert-base-uncased""": """https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/vocab.txt""",
    },
    """tokenizer_file""": {
        """unc-nlp/lxmert-base-uncased""": (
            """https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/tokenizer.json"""
        ),
    },
}

lowercase      :   List[Any]      =       {
    """unc-nlp/lxmert-base-uncased""": 512,
}

lowercase      :   Tuple      =       {
    """unc-nlp/lxmert-base-uncased""": {"""do_lower_case""": True},
}







class 					_a (a__	):





     '''simple docstring'''



     lowerCAmelCase_  :     Optional[int]          =					VOCAB_FILES_NAMES
     lowerCAmelCase_  :     Optional[int]          =					PRETRAINED_VOCAB_FILES_MAP
     lowerCAmelCase_  :     Any          =					PRETRAINED_INIT_CONFIGURATION
     lowerCAmelCase_  :     Optional[int]          =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     lowerCAmelCase_  :     Optional[int]          =					LxmertTokenizer







     def __init__(				self							,__a=None							,__a=None							,__a=True							,__a="[UNK]"							,__a="[SEP]"							,__a="[PAD]"							,__a="[CLS]"							,__a="[MASK]"							,__a=True							,__a=None							,**__a							,)      ->				str:
          super().__init__(
              __a							,tokenizer_file=__a							,do_lower_case=__a							,unk_token=__a							,sep_token=__a							,pad_token=__a							,cls_token=__a							,mask_token=__a							,tokenize_chinese_chars=__a							,strip_accents=__a							,**__a							,)

          snake_case			:      List[Any]        =			json.loads(self.backend_tokenizer.normalizer.__getstate__()							)
          if (
              normalizer_state.get("""lowercase"""							,__a							) != do_lower_case
              or normalizer_state.get("""strip_accents"""							,__a							) != strip_accents
              or normalizer_state.get("""handle_chinese_chars"""							,__a							) != tokenize_chinese_chars
          ):
               snake_case			:      Union[str, Any]        =			getattr(__a							,normalizer_state.pop("""type"""							)							)
               snake_case			:      Optional[int]        =			do_lower_case
               snake_case			:      Optional[int]        =			strip_accents
               snake_case			:      List[Any]        =			tokenize_chinese_chars
               snake_case			:      Union[str, Any]        =			normalizer_class(**__a							)

          snake_case			:      str        =			do_lower_case







     def 			snake_case_      (				self							,__a							,__a=None							)      ->				Optional[int]:
          snake_case			:      Tuple        =			[self.cls_token_id] + token_ids_a + [self.sep_token_id]

          if token_ids_a:
               output += token_ids_a + [self.sep_token_id]

          return output







     def 			snake_case_      (				self							,__a							,__a = None							)      ->				List[int]:
          snake_case			:      int        =			[self.sep_token_id]
          snake_case			:      Any        =			[self.cls_token_id]
          if token_ids_a is None:
               return len(cls + token_ids_a + sep							) * [0]
          return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]







     def 			snake_case_      (				self							,__a							,__a = None							)      ->				Tuple[str]:
          snake_case			:      Optional[int]        =			self._tokenizer.model.save(__a							,name=__a							)
          return tuple(__a							)



