





'''simple docstring'''




import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / '''utils'''))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
  from test_module.custom_tokenization_fast import CustomTokenizerFast


class 					SCREAMING_SNAKE_CASE       (unittest.TestCase				):


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					str						=			mock.Mock()
          __A		:					List[Any]						=			500
          __A		:					Optional[int]						=			{}
          __A		:					str						=			HTTPError
          __A		:					Any						=			{}

          # Download this model to make sure it's in the cache.
          __A		:					Tuple						=			BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert')

          # Under the mock environment we get a 500 error when trying to reach the tokenizer.
          with mock.patch('requests.Session.request'		,      return_value=_UpperCAmelCase) as mock_head:
               __A		:					Optional[Any]						=			BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert')
               # This check we did call the fake head request
               mock_head.assert_called()


     @require_tokenizers
     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					Dict						=			mock.Mock()
          __A		:					Union[str, Any]						=			500
          __A		:					Any						=			{}
          __A		:					Tuple						=			HTTPError
          __A		:					Tuple						=			{}

          # Download this model to make sure it's in the cache.
          __A		:					Optional[int]						=			GPTaTokenizerFast.from_pretrained('gpt2')

          # Under the mock environment we get a 500 error when trying to reach the tokenizer.
          with mock.patch('requests.Session.request'		,      return_value=_UpperCAmelCase) as mock_head:
               __A		:					str						=			GPTaTokenizerFast.from_pretrained('gpt2')
               # This check we did call the fake head request
               mock_head.assert_called()


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          try:
               __A		:					Any						=			tempfile.mktemp()
               with open(_UpperCAmelCase		,      'wb') as f:
                    http_get('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'		,      _UpperCAmelCase)

               __A		:					Tuple						=			AlbertTokenizer.from_pretrained(_UpperCAmelCase)
          finally:
               os.remove(_UpperCAmelCase)

          # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
          # the current folder and have the right name.
          if os.path.isfile('tokenizer.json'):
               # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
               return
          try:
               with open('tokenizer.json'		,      'wb') as f:
                    http_get('https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'		,      _UpperCAmelCase)
               __A		:					Any						=			AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-gpt2')
               # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
               self.assertEqual(tokenizer.vocab_size		,      1000)
               # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

          finally:
               os.remove('tokenizer.json')


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					List[str]						=			AlbertTokenizer.from_pretrained('https://huggingface.co/albert-base-v1/resolve/main/spiece.model')




@is_staging_test
class 					SCREAMING_SNAKE_CASE       (unittest.TestCase				):
     lowerCAmelCase						     =			['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''bla''', '''blou''']


     @classmethod
     def 				SCREAMING_SNAKE_CASE    (		cls):

          '''simple docstring'''






          __A		:					List[Any]						=			TOKEN
          HfFolder.save_token(_UpperCAmelCase)


     @classmethod
     def 				SCREAMING_SNAKE_CASE    (		cls):

          '''simple docstring'''






          try:
               delete_repo(token=cls._token		,      repo_id='test-tokenizer')
          except HTTPError:
               pass

          try:
               delete_repo(token=cls._token		,      repo_id='valid_org/test-tokenizer-org')
          except HTTPError:
               pass

          try:
               delete_repo(token=cls._token		,      repo_id='test-dynamic-tokenizer')
          except HTTPError:
               pass


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          with tempfile.TemporaryDirectory() as tmp_dir:
               __A		:					Dict						=			os.path.join(_UpperCAmelCase		,      'vocab.txt')
               with open(_UpperCAmelCase		,      'w'		,      encoding='utf-8') as vocab_writer:
                    vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]))
               __A		:					Optional[Any]						=			BertTokenizer(_UpperCAmelCase)

          tokenizer.push_to_hub('test-tokenizer'		,      use_auth_token=self._token)
          __A		:					Dict						=			BertTokenizer.from_pretrained(F'{USER}/test-tokenizer')
          self.assertDictEqual(new_tokenizer.vocab		,      tokenizer.vocab)

          # Reset repo
          delete_repo(token=self._token		,      repo_id='test-tokenizer')

          # Push to hub via save_pretrained
          with tempfile.TemporaryDirectory() as tmp_dir:
               tokenizer.save_pretrained(_UpperCAmelCase		,      repo_id='test-tokenizer'		,      push_to_hub=_UpperCAmelCase		,      use_auth_token=self._token)

          __A		:					Any						=			BertTokenizer.from_pretrained(F'{USER}/test-tokenizer')
          self.assertDictEqual(new_tokenizer.vocab		,      tokenizer.vocab)


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          with tempfile.TemporaryDirectory() as tmp_dir:
               __A		:					Optional[int]						=			os.path.join(_UpperCAmelCase		,      'vocab.txt')
               with open(_UpperCAmelCase		,      'w'		,      encoding='utf-8') as vocab_writer:
                    vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]))
               __A		:					List[Any]						=			BertTokenizer(_UpperCAmelCase)

          tokenizer.push_to_hub('valid_org/test-tokenizer-org'		,      use_auth_token=self._token)
          __A		:					List[Any]						=			BertTokenizer.from_pretrained('valid_org/test-tokenizer-org')
          self.assertDictEqual(new_tokenizer.vocab		,      tokenizer.vocab)

          # Reset repo
          delete_repo(token=self._token		,      repo_id='valid_org/test-tokenizer-org')

          # Push to hub via save_pretrained
          with tempfile.TemporaryDirectory() as tmp_dir:
               tokenizer.save_pretrained(
                   _UpperCAmelCase		,      repo_id='valid_org/test-tokenizer-org'		,      push_to_hub=_UpperCAmelCase		,      use_auth_token=self._token)

          __A		:					str						=			BertTokenizer.from_pretrained('valid_org/test-tokenizer-org')
          self.assertDictEqual(new_tokenizer.vocab		,      tokenizer.vocab)


     @require_tokenizers
     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          CustomTokenizer.register_for_auto_class()
          with tempfile.TemporaryDirectory() as tmp_dir:
               __A		:					str						=			os.path.join(_UpperCAmelCase		,      'vocab.txt')
               with open(_UpperCAmelCase		,      'w'		,      encoding='utf-8') as vocab_writer:
                    vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]))
               __A		:					Any						=			CustomTokenizer(_UpperCAmelCase)

          # No fast custom tokenizer
          tokenizer.push_to_hub('test-dynamic-tokenizer'		,      use_auth_token=self._token)

          __A		:					List[Any]						=			AutoTokenizer.from_pretrained(F'{USER}/test-dynamic-tokenizer'		,      trust_remote_code=_UpperCAmelCase)
          # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
          self.assertEqual(tokenizer.__class__.__name__		,      'CustomTokenizer')

          # Fast and slow custom tokenizer
          CustomTokenizerFast.register_for_auto_class()
          with tempfile.TemporaryDirectory() as tmp_dir:
               __A		:					List[str]						=			os.path.join(_UpperCAmelCase		,      'vocab.txt')
               with open(_UpperCAmelCase		,      'w'		,      encoding='utf-8') as vocab_writer:
                    vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]))

               __A		:					Tuple						=			BertTokenizerFast.from_pretrained(_UpperCAmelCase)
               bert_tokenizer.save_pretrained(_UpperCAmelCase)
               __A		:					Any						=			CustomTokenizerFast.from_pretrained(_UpperCAmelCase)

          tokenizer.push_to_hub('test-dynamic-tokenizer'		,      use_auth_token=self._token)

          __A		:					Tuple						=			AutoTokenizer.from_pretrained(F'{USER}/test-dynamic-tokenizer'		,      trust_remote_code=_UpperCAmelCase)
          # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
          self.assertEqual(tokenizer.__class__.__name__		,      'CustomTokenizerFast')
          __A		:					Union[str, Any]						=			AutoTokenizer.from_pretrained(
              F'{USER}/test-dynamic-tokenizer'		,      use_fast=_UpperCAmelCase		,      trust_remote_code=_UpperCAmelCase)
          # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
          self.assertEqual(tokenizer.__class__.__name__		,      'CustomTokenizer')




class 					SCREAMING_SNAKE_CASE       (unittest.TestCase				):


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					str						=			Trie()
          trie.add('Hello 友達')
          self.assertEqual(trie.data		,      {'H': {'e': {'l': {'l': {'o': {' ': {'友': {'達': {'': 1}}}}}}}}})
          trie.add('Hello')
          trie.data
          self.assertEqual(trie.data		,      {'H': {'e': {'l': {'l': {'o': {'': 1, ' ': {'友': {'達': {'': 1}}}}}}}}})


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					Any						=			Trie()
          self.assertEqual(trie.split('[CLS] This is a extra_id_100')		,      ['[CLS] This is a extra_id_100'])
          trie.add('[CLS]')
          trie.add('extra_id_1')
          trie.add('extra_id_100')
          self.assertEqual(trie.split('[CLS] This is a extra_id_100')		,      ['[CLS]', ' This is a ', 'extra_id_100'])


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					Union[str, Any]						=			Trie()
          trie.add('A')
          self.assertEqual(trie.split('ABC')		,      ['A', 'BC'])
          self.assertEqual(trie.split('BCA')		,      ['BC', 'A'])


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					List[str]						=			Trie()
          trie.add('TOKEN]')
          trie.add('[SPECIAL_TOKEN]')
          self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]')		,      ['This is something ', '[SPECIAL_TOKEN]'])


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					Tuple						=			Trie()
          trie.add('A')
          trie.add('P')
          trie.add('[SPECIAL_TOKEN]')
          self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]')		,      ['This is something ', '[SPECIAL_TOKEN]'])


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					Optional[int]						=			Trie()
          trie.add('AB')
          trie.add('B')
          trie.add('C')
          self.assertEqual(trie.split('ABC')		,      ['AB', 'C'])


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					Optional[Any]						=			Trie()
          trie.add('ABC')
          trie.add('B')
          trie.add('CD')
          self.assertEqual(trie.split('ABCD')		,      ['ABC', 'D'])


     def 				SCREAMING_SNAKE_CASE    (		self):

          '''simple docstring'''






          __A		:					Optional[int]						=			Trie()
          __A		:					Optional[int]						=			trie.cut_text('ABC'		,      [0, 0, 2, 1, 2, 3])
          self.assertEqual(_UpperCAmelCase		,      ['AB', 'C'])






'''simple docstring'''




from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase__       : List[Any]         =       logging.get_logger(__name__)

lowercase__       : Optional[int]         =       {
    '''unc-nlp/lxmert-base-uncased''': '''https://huggingface.co/unc-nlp/lxmert-base-uncased/resolve/main/config.json''',
}


class 					SCREAMING_SNAKE_CASE       (a__				):
     lowerCAmelCase						     =			'''lxmert'''
     lowerCAmelCase						     =			{}


     def __init__(		self		,      _UpperCAmelCase=3_0522		,      _UpperCAmelCase=768		,      _UpperCAmelCase=12		,      _UpperCAmelCase=9500		,      _UpperCAmelCase=1600		,      _UpperCAmelCase=400		,      _UpperCAmelCase=3072		,      _UpperCAmelCase="gelu"		,      _UpperCAmelCase=0.1		,      _UpperCAmelCase=0.1		,      _UpperCAmelCase=512		,      _UpperCAmelCase=2		,      _UpperCAmelCase=0.02		,      _UpperCAmelCase=1e-1_2		,      _UpperCAmelCase=9		,      _UpperCAmelCase=5		,      _UpperCAmelCase=5		,      _UpperCAmelCase=2048		,      _UpperCAmelCase=4		,      _UpperCAmelCase=6.67		,      _UpperCAmelCase=True		,      _UpperCAmelCase=True		,      _UpperCAmelCase=True		,      _UpperCAmelCase=True		,      _UpperCAmelCase=True		,      _UpperCAmelCase=True		,      _UpperCAmelCase=True		,      **_UpperCAmelCase		,      ):

          '''simple docstring'''






          __A		:					Tuple						=			vocab_size
          __A		:					int						=			hidden_size
          __A		:					str						=			num_attention_heads
          __A		:					Tuple						=			hidden_act
          __A		:					int						=			intermediate_size
          __A		:					str						=			hidden_dropout_prob
          __A		:					Optional[int]						=			attention_probs_dropout_prob
          __A		:					Optional[Any]						=			max_position_embeddings
          __A		:					Tuple						=			type_vocab_size
          __A		:					Optional[int]						=			initializer_range
          __A		:					Any						=			layer_norm_eps
          __A		:					Optional[Any]						=			num_qa_labels
          __A		:					Optional[int]						=			num_object_labels
          __A		:					Any						=			num_attr_labels
          __A		:					Union[str, Any]						=			l_layers
          __A		:					Optional[int]						=			x_layers
          __A		:					List[Any]						=			r_layers
          __A		:					Tuple						=			visual_feat_dim
          __A		:					Tuple						=			visual_pos_dim
          __A		:					Optional[int]						=			visual_loss_normalizer
          __A		:					int						=			task_matched
          __A		:					List[Any]						=			task_mask_lm
          __A		:					Optional[Any]						=			task_obj_predict
          __A		:					str						=			task_qa
          __A		:					List[Any]						=			visual_obj_loss
          __A		:					Optional[Any]						=			visual_attr_loss
          __A		:					Union[str, Any]						=			visual_feat_loss
          __A		:					Union[str, Any]						=			{'vision': r_layers, 'cross_encoder': x_layers, 'language': l_layers}
          super().__init__(**_UpperCAmelCase)