




'''simple docstring'''




import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / """utils"""))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
     from test_module.custom_tokenization_fast import CustomTokenizerFast


class     __UpperCamelCase			(					unittest.TestCase							):






       def 	lowercase__	(				self    ):



              """simple docstring"""







              lowerCamelCase_				=mock.Mock()
              lowerCamelCase_				=500
              lowerCamelCase_				={}
              lowerCamelCase_				=HTTPError
              lowerCamelCase_				={}

              # Download this model to make sure it's in the cache.
              lowerCamelCase_				=BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''    )

              # Under the mock environment we get a 500 error when trying to reach the tokenizer.
              with mock.patch('''requests.Session.request''',						return_value=_UpperCAmelCase    ) as mock_head:
                     lowerCamelCase_				=BertTokenizer.from_pretrained('''hf-internal-testing/tiny-random-bert'''    )
                     # This check we did call the fake head request
                     mock_head.assert_called()






       @require_tokenizers
       def 	lowercase__	(				self    ):



              """simple docstring"""







              lowerCamelCase_				=mock.Mock()
              lowerCamelCase_				=500
              lowerCamelCase_				={}
              lowerCamelCase_				=HTTPError
              lowerCamelCase_				={}

              # Download this model to make sure it's in the cache.
              lowerCamelCase_				=GPTaTokenizerFast.from_pretrained('''gpt2'''    )

              # Under the mock environment we get a 500 error when trying to reach the tokenizer.
              with mock.patch('''requests.Session.request''',						return_value=_UpperCAmelCase    ) as mock_head:
                     lowerCamelCase_				=GPTaTokenizerFast.from_pretrained('''gpt2'''    )
                     # This check we did call the fake head request
                     mock_head.assert_called()






       def 	lowercase__	(				self    ):



              """simple docstring"""







              try:
                     lowerCamelCase_				=tempfile.mktemp()
                     with open(_UpperCAmelCase,						'''wb'''    ) as f:
                            http_get('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model''',						_UpperCAmelCase    )

                     lowerCamelCase_				=AlbertTokenizer.from_pretrained(_UpperCAmelCase    )
              finally:
                     os.remove(_UpperCAmelCase    )

              # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
              # the current folder and have the right name.
              if os.path.isfile('''tokenizer.json'''    ):
                     # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
                     return
              try:
                     with open('''tokenizer.json''',						'''wb'''    ) as f:
                            http_get('''https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json''',						_UpperCAmelCase    )
                     lowerCamelCase_				=AutoTokenizer.from_pretrained('''hf-internal-testing/tiny-random-gpt2'''    )
                     # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
                     self.assertEqual(tokenizer.vocab_size,						1_000    )
                     # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

              finally:
                     os.remove('''tokenizer.json'''    )






       def 	lowercase__	(				self    ):



              """simple docstring"""







              lowerCamelCase_				=AlbertTokenizer.from_pretrained('''https://huggingface.co/albert-base-v1/resolve/main/spiece.model'''    )





@is_staging_test
class     __UpperCamelCase			(					unittest.TestCase							):
       lowercase      :       Any 		=['[UNK]', '[CLS]', '[SEP]', '[PAD]', '[MASK]', 'bla', 'blou']






       @classmethod
       def 	lowercase__	(				cls    ):



              """simple docstring"""







              lowerCamelCase_				=TOKEN
              HfFolder.save_token(_UpperCAmelCase    )






       @classmethod
       def 	lowercase__	(				cls    ):



              """simple docstring"""







              try:
                     delete_repo(token=cls._token,						repo_id='''test-tokenizer'''    )
              except HTTPError:
                     pass

              try:
                     delete_repo(token=cls._token,						repo_id='''valid_org/test-tokenizer-org'''    )
              except HTTPError:
                     pass

              try:
                     delete_repo(token=cls._token,						repo_id='''test-dynamic-tokenizer'''    )
              except HTTPError:
                     pass






       def 	lowercase__	(				self    ):



              """simple docstring"""







              with tempfile.TemporaryDirectory() as tmp_dir:
                     lowerCamelCase_				=os.path.join(_UpperCAmelCase,						'''vocab.txt'''    )
                     with open(_UpperCAmelCase,						'''w''',						encoding='''utf-8'''    ) as vocab_writer:
                            vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]    )    )
                     lowerCamelCase_				=BertTokenizer(_UpperCAmelCase    )

              tokenizer.push_to_hub('''test-tokenizer''',						use_auth_token=self._token    )
              lowerCamelCase_				=BertTokenizer.from_pretrained(f'''{USER}/test-tokenizer'''    )
              self.assertDictEqual(new_tokenizer.vocab,						tokenizer.vocab    )

              # Reset repo
              delete_repo(token=self._token,						repo_id='''test-tokenizer'''    )

              # Push to hub via save_pretrained
              with tempfile.TemporaryDirectory() as tmp_dir:
                     tokenizer.save_pretrained(_UpperCAmelCase,						repo_id='''test-tokenizer''',						push_to_hub=_UpperCAmelCase,						use_auth_token=self._token    )

              lowerCamelCase_				=BertTokenizer.from_pretrained(f'''{USER}/test-tokenizer'''    )
              self.assertDictEqual(new_tokenizer.vocab,						tokenizer.vocab    )






       def 	lowercase__	(				self    ):



              """simple docstring"""







              with tempfile.TemporaryDirectory() as tmp_dir:
                     lowerCamelCase_				=os.path.join(_UpperCAmelCase,						'''vocab.txt'''    )
                     with open(_UpperCAmelCase,						'''w''',						encoding='''utf-8'''    ) as vocab_writer:
                            vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]    )    )
                     lowerCamelCase_				=BertTokenizer(_UpperCAmelCase    )

              tokenizer.push_to_hub('''valid_org/test-tokenizer-org''',						use_auth_token=self._token    )
              lowerCamelCase_				=BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''    )
              self.assertDictEqual(new_tokenizer.vocab,						tokenizer.vocab    )

              # Reset repo
              delete_repo(token=self._token,						repo_id='''valid_org/test-tokenizer-org'''    )

              # Push to hub via save_pretrained
              with tempfile.TemporaryDirectory() as tmp_dir:
                     tokenizer.save_pretrained(
                         _UpperCAmelCase,						repo_id='''valid_org/test-tokenizer-org''',						push_to_hub=_UpperCAmelCase,						use_auth_token=self._token    )

              lowerCamelCase_				=BertTokenizer.from_pretrained('''valid_org/test-tokenizer-org'''    )
              self.assertDictEqual(new_tokenizer.vocab,						tokenizer.vocab    )






       @require_tokenizers
       def 	lowercase__	(				self    ):



              """simple docstring"""







              CustomTokenizer.register_for_auto_class()
              with tempfile.TemporaryDirectory() as tmp_dir:
                     lowerCamelCase_				=os.path.join(_UpperCAmelCase,						'''vocab.txt'''    )
                     with open(_UpperCAmelCase,						'''w''',						encoding='''utf-8'''    ) as vocab_writer:
                            vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]    )    )
                     lowerCamelCase_				=CustomTokenizer(_UpperCAmelCase    )

              # No fast custom tokenizer
              tokenizer.push_to_hub('''test-dynamic-tokenizer''',						use_auth_token=self._token    )

              lowerCamelCase_				=AutoTokenizer.from_pretrained(f'''{USER}/test-dynamic-tokenizer''',						trust_remote_code=_UpperCAmelCase    )
              # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
              self.assertEqual(tokenizer.__class__.__name__,						'''CustomTokenizer'''    )

              # Fast and slow custom tokenizer
              CustomTokenizerFast.register_for_auto_class()
              with tempfile.TemporaryDirectory() as tmp_dir:
                     lowerCamelCase_				=os.path.join(_UpperCAmelCase,						'''vocab.txt'''    )
                     with open(_UpperCAmelCase,						'''w''',						encoding='''utf-8'''    ) as vocab_writer:
                            vocab_writer.write(''''''.join([x + '''\n''' for x in self.vocab_tokens]    )    )

                     lowerCamelCase_				=BertTokenizerFast.from_pretrained(_UpperCAmelCase    )
                     bert_tokenizer.save_pretrained(_UpperCAmelCase    )
                     lowerCamelCase_				=CustomTokenizerFast.from_pretrained(_UpperCAmelCase    )

              tokenizer.push_to_hub('''test-dynamic-tokenizer''',						use_auth_token=self._token    )

              lowerCamelCase_				=AutoTokenizer.from_pretrained(f'''{USER}/test-dynamic-tokenizer''',						trust_remote_code=_UpperCAmelCase    )
              # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
              self.assertEqual(tokenizer.__class__.__name__,						'''CustomTokenizerFast'''    )
              lowerCamelCase_				=AutoTokenizer.from_pretrained(
                  f'''{USER}/test-dynamic-tokenizer''',						use_fast=_UpperCAmelCase,						trust_remote_code=_UpperCAmelCase    )
              # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
              self.assertEqual(tokenizer.__class__.__name__,						'''CustomTokenizer'''    )





class     __UpperCamelCase			(					unittest.TestCase							):






       def 	lowercase__	(				self    ):



              """simple docstring"""







              lowerCamelCase_				=Trie()
              trie.add('''Hello 友達'''    )
              self.assertEqual(trie.data,						{'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}    )
              trie.add('''Hello'''    )
              trie.data
              self.assertEqual(trie.data,						{'''H''': {'''e''': {'''l''': {'''l''': {'''o''': {'''''': 1, ''' ''': {'''友''': {'''達''': {'''''': 1}}}}}}}}}    )






       def 	lowercase__	(				self    ):



              """simple docstring"""







              lowerCamelCase_				=Trie()
              self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''    ),						['''[CLS] This is a extra_id_100''']    )
              trie.add('''[CLS]'''    )
              trie.add('''extra_id_1'''    )
              trie.add('''extra_id_100'''    )
              self.assertEqual(trie.split('''[CLS] This is a extra_id_100'''    ),						['''[CLS]''', ''' This is a ''', '''extra_id_100''']    )






       def 	lowercase__	(				self    ):



              """simple docstring"""







              lowerCamelCase_				=Trie()
              trie.add('''A'''    )
              self.assertEqual(trie.split('''ABC'''    ),						['''A''', '''BC''']    )
              self.assertEqual(trie.split('''BCA'''    ),						['''BC''', '''A''']    )






       def 	lowercase__	(				self    ):



              """simple docstring"""







              lowerCamelCase_				=Trie()
              trie.add('''TOKEN]'''    )
              trie.add('''[SPECIAL_TOKEN]'''    )
              self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''    ),						['''This is something ''', '''[SPECIAL_TOKEN]''']    )






       def 	lowercase__	(				self    ):



              """simple docstring"""







              lowerCamelCase_				=Trie()
              trie.add('''A'''    )
              trie.add('''P'''    )
              trie.add('''[SPECIAL_TOKEN]'''    )
              self.assertEqual(trie.split('''This is something [SPECIAL_TOKEN]'''    ),						['''This is something ''', '''[SPECIAL_TOKEN]''']    )






       def 	lowercase__	(				self    ):



              """simple docstring"""







              lowerCamelCase_				=Trie()
              trie.add('''AB'''    )
              trie.add('''B'''    )
              trie.add('''C'''    )
              self.assertEqual(trie.split('''ABC'''    ),						['''AB''', '''C''']    )






       def 	lowercase__	(				self    ):



              """simple docstring"""







              lowerCamelCase_				=Trie()
              trie.add('''ABC'''    )
              trie.add('''B'''    )
              trie.add('''CD'''    )
              self.assertEqual(trie.split('''ABCD'''    ),						['''ABC''', '''D''']    )






       def 	lowercase__	(				self    ):



              """simple docstring"""







              lowerCamelCase_				=Trie()
              lowerCamelCase_				=trie.cut_text('''ABC''',						[0, 0, 2, 1, 2, 3]    )
              self.assertEqual(_UpperCAmelCase,						['''AB''', '''C''']    )











'''simple docstring'''




from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase__       : List[Any]         =       logging.get_logger(__name__)

lowercase__       : Optional[int]         =       {
    '''unc-nlp/lxmert-base-uncased''': '''https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/config.json''',
}


class 					SCREAMING_SNAKE_CASE       (a__				):
     lowerCAmelCase						     =			'''lxmert'''
     lowerCAmelCase						     =			{}


     def __init__(		self		,      _UpperCAmelCase=3_0522		,      _UpperCAmelCase=768		,      _UpperCAmelCase=12		,      _UpperCAmelCase=9500		,      _UpperCAmelCase=1600		,      _UpperCAmelCase=400		,      _UpperCAmelCase=3072		,      _UpperCAmelCase="gelu"		,      _UpperCAmelCase=0.1		,      _UpperCAmelCase=0.1		,      _UpperCAmelCase=512		,      _UpperCAmelCase=2		,      _UpperCAmelCase=0.02		,      _UpperCAmelCase=1e-1_2		,      _UpperCAmelCase=9		,      _UpperCAmelCase=5		,      _UpperCAmelCase=5		,      _UpperCAmelCase=2048		,      _UpperCAmelCase=4		,      _UpperCAmelCase=6.67		,      _UpperCAmelCase=True		,      _UpperCAmelCase=True		,      _UpperCAmelCase=True		,      _UpperCAmelCase=True		,      _UpperCAmelCase=True		,      _UpperCAmelCase=True		,      _UpperCAmelCase=True		,      **_UpperCAmelCase		,      ):

          '''simple docstring'''






          __A		:					Tuple						=			vocab_size
          __A		:					int						=			hidden_size
          __A		:					str						=			num_attention_heads
          __A		:					Tuple						=			hidden_act
          __A		:					int						=			intermediate_size
          __A		:					str						=			hidden_dropout_prob
          __A		:					Optional[int]						=			attention_probs_dropout_prob
          __A		:					Optional[Any]						=			max_position_embeddings
          __A		:					Tuple						=			type_vocab_size
          __A		:					Optional[int]						=			initializer_range
          __A		:					Any						=			layer_norm_eps
          __A		:					Optional[Any]						=			num_qa_labels
          __A		:					Optional[int]						=			num_object_labels
          __A		:					Any						=			num_attr_labels
          __A		:					Union[str, Any]						=			l_layers
          __A		:					Optional[int]						=			x_layers
          __A		:					List[Any]						=			r_layers
          __A		:					Tuple						=			visual_feat_dim
          __A		:					Tuple						=			visual_pos_dim
          __A		:					Optional[int]						=			visual_loss_normalizer
          __A		:					int						=			task_matched
          __A		:					List[Any]						=			task_mask_lm
          __A		:					Optional[Any]						=			task_obj_predict
          __A		:					str						=			task_qa
          __A		:					List[Any]						=			visual_obj_loss
          __A		:					Optional[Any]						=			visual_attr_loss
          __A		:					Union[str, Any]						=			visual_feat_loss
          __A		:					Union[str, Any]						=			{'vision': r_layers, 'cross_encoder': x_layers, 'language': l_layers}
          super().__init__(**_UpperCAmelCase)