






import inspect
import unittest

from transformers import ViTMSNConfig
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
			import torch
			from torch import nn

			from transformers import ViTMSNForImageClassification, ViTMSNModel
			from transformers.models.vit_msn.modeling_vit_msn import VIT_MSN_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
			from PIL import Image

			from transformers import ViTImageProcessor




class     _lowerCAmelCase:





							"""simple docstring"""







							def __init__( self  ,      _lowerCamelCase  ,      _lowerCamelCase=1_3  ,      _lowerCamelCase=3_0  ,      _lowerCamelCase=2  ,      _lowerCamelCase=3  ,      _lowerCamelCase=True  ,      _lowerCamelCase=True  ,      _lowerCamelCase=3_2  ,      _lowerCamelCase=5  ,      _lowerCamelCase=4  ,      _lowerCamelCase=3_7  ,      _lowerCamelCase="gelu"  ,      _lowerCamelCase=0.1  ,      _lowerCamelCase=0.1  ,      _lowerCamelCase=1_0  ,      _lowerCamelCase=0.0_2  ,      _lowerCamelCase=None  ,      ):
									UpperCamelCase_: Optional[Any]   				=    parent
									UpperCamelCase_: Union[str, Any]   				=    batch_size
									UpperCamelCase_: Optional[int]   				=    image_size
									UpperCamelCase_: List[str]   				=    patch_size
									UpperCamelCase_: Optional[int]   				=    num_channels
									UpperCamelCase_: Tuple   				=    is_training
									UpperCamelCase_: Optional[int]   				=    use_labels
									UpperCamelCase_: str   				=    hidden_size
									UpperCamelCase_: Optional[Any]   				=    num_hidden_layers
									UpperCamelCase_: List[str]   				=    num_attention_heads
									UpperCamelCase_: List[str]   				=    intermediate_size
									UpperCamelCase_: Optional[Any]   				=    hidden_act
									UpperCamelCase_: List[Any]   				=    hidden_dropout_prob
									UpperCamelCase_: List[Any]   				=    attention_probs_dropout_prob
									UpperCamelCase_: int   				=    type_sequence_label_size
									UpperCamelCase_: int   				=    initializer_range
									UpperCamelCase_: Tuple   				=    scope

									# in ViT MSN, the seq length equals the number of patches + 1 (we add 1 for the [CLS] token)
									UpperCamelCase_: List[Any]   				=    (image_size // patch_size) ** 2
									UpperCamelCase_: int   				=    num_patches + 1





							def       _a   ( self							):
									UpperCamelCase_: List[Any]   				=    floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]							)

									UpperCamelCase_: List[Any]   				=    None
									if self.use_labels:
											UpperCamelCase_: List[str]   				=    ids_tensor([self.batch_size]  ,      self.type_sequence_label_size							)

									UpperCamelCase_: List[Any]   				=    self.get_config()

									return config, pixel_values, labels





							def       _a   ( self							):
									return ViTMSNConfig(
									    image_size=self.image_size  ,      patch_size=self.patch_size  ,      num_channels=self.num_channels  ,      hidden_size=self.hidden_size  ,      num_hidden_layers=self.num_hidden_layers  ,      num_attention_heads=self.num_attention_heads  ,      intermediate_size=self.intermediate_size  ,      hidden_act=self.hidden_act  ,      hidden_dropout_prob=self.hidden_dropout_prob  ,      attention_probs_dropout_prob=self.attention_probs_dropout_prob  ,      initializer_range=self.initializer_range  ,      )





							def       _a   ( self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase							):
									UpperCamelCase_: Optional[int]   				=    ViTMSNModel(config=_lowerCamelCase							)
									model.to(_lowerCamelCase							)
									model.eval()
									UpperCamelCase_: Any   				=    model(_lowerCamelCase							)
									self.parent.assertEqual(result.last_hidden_state.shape  ,      (self.batch_size, self.seq_length, self.hidden_size)							)





							def       _a   ( self  ,      _lowerCamelCase  ,      _lowerCamelCase  ,      _lowerCamelCase							):
									UpperCamelCase_: Union[str, Any]   				=    self.type_sequence_label_size
									UpperCamelCase_: List[Any]   				=    ViTMSNForImageClassification(_lowerCamelCase							)
									model.to(_lowerCamelCase							)
									model.eval()
									UpperCamelCase_: Tuple   				=    model(_lowerCamelCase  ,      labels=_lowerCamelCase							)
									print('Pixel and labels shape: {pixel_values.shape}, {labels.shape}'							)
									print('Labels: {labels}'							)
									self.parent.assertEqual(result.logits.shape  ,      (self.batch_size, self.type_sequence_label_size)							)

									# test greyscale images
									UpperCamelCase_: List[str]   				=    1
									UpperCamelCase_: Optional[Any]   				=    ViTMSNForImageClassification(_lowerCamelCase							)
									model.to(_lowerCamelCase							)
									model.eval()

									UpperCamelCase_: Dict   				=    floats_tensor([self.batch_size, 1, self.image_size, self.image_size]							)
									UpperCamelCase_: Any   				=    model(_lowerCamelCase							)
									self.parent.assertEqual(result.logits.shape  ,      (self.batch_size, self.type_sequence_label_size)							)





							def       _a   ( self							):
									UpperCamelCase_: Optional[int]   				=    self.prepare_config_and_inputs()
									UpperCamelCase_: str   				=    config_and_inputs
									UpperCamelCase_: Dict   				=    {'pixel_values': pixel_values}
									return config, inputs_dict







@require_torch
class     _lowerCAmelCase(	UpperCAmelCase_							,     UpperCAmelCase_							,     unittest.TestCase ):





							"""simple docstring"""







							a    :		str									=(ViTMSNModel, ViTMSNForImageClassification) if is_torch_available() else ()
							a    :		Union[str, Any]									=(
							    {'''feature-extraction''': ViTMSNModel, '''image-classification''': ViTMSNForImageClassification}
							    if is_torch_available()
							    else {}
							)

							a    :		Union[str, Any]									=False
							a    :		Dict									=False
							a    :		Union[str, Any]									=False
							a    :		List[str]									=False





							def       _a   ( self							):
									UpperCamelCase_: str   				=    ViTMSNModelTester(self							)
									UpperCamelCase_: Tuple   				=    ConfigTester(self  ,      config_class=_lowerCamelCase  ,      has_text_modality=_lowerCamelCase  ,      hidden_size=3_7							)





							def       _a   ( self							):
									self.config_tester.run_common_tests()





							@unittest.skip(reason='ViTMSN does not use inputs_embeds'							)
							def       _a   ( self							):
									pass





							def       _a   ( self							):
									UpperCamelCase_: List[str]   				=    self.model_tester.prepare_config_and_inputs_for_common()

									for model_class in self.all_model_classes:
											UpperCamelCase_: int   				=    model_class(_lowerCamelCase							)
											self.assertIsInstance(model.get_input_embeddings()  ,      (nn.Module)							)
											UpperCamelCase_: Any   				=    model.get_output_embeddings()
											self.assertTrue(x is None or isinstance(_lowerCamelCase  ,      nn.Linear							)							)





							def       _a   ( self							):
									UpperCamelCase_: Dict   				=    self.model_tester.prepare_config_and_inputs_for_common()

									for model_class in self.all_model_classes:
											UpperCamelCase_: Tuple   				=    model_class(_lowerCamelCase							)
											UpperCamelCase_: List[Any]   				=    inspect.signature(model.forward							)
											# signature.parameters is an OrderedDict => so arg_names order is deterministic
											UpperCamelCase_: Dict   				=    [*signature.parameters.keys()]

											UpperCamelCase_: Tuple   				=    ['pixel_values']
											self.assertListEqual(arg_names[:1]  ,      _lowerCamelCase							)





							def       _a   ( self							):
									UpperCamelCase_: Dict   				=    self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_model(*_lowerCamelCase							)





							def       _a   ( self							):
									UpperCamelCase_: str   				=    self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_image_classification(*_lowerCamelCase							)





							@slow
							def       _a   ( self							):
									for model_name in VIT_MSN_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
											UpperCamelCase_: str   				=    ViTMSNModel.from_pretrained(_lowerCamelCase							)
											self.assertIsNotNone(_lowerCamelCase							)







def   snake_case ()	->    str:
		UpperCamelCase_: List[Any]   				=    Image.open('./tests/fixtures/tests_samples/COCO/000000039769.png'  )
		return image




@require_torch
@require_vision
class     _lowerCAmelCase(	unittest.TestCase ):





							"""simple docstring"""







							@cached_property
							def       _a   ( self							):
									return ViTImageProcessor.from_pretrained('facebook/vit-msn-small'							) if is_vision_available() else None





							@slow
							def       _a   ( self							):
									torch.manual_seed(2							)
									UpperCamelCase_: str   				=    ViTMSNForImageClassification.from_pretrained('facebook/vit-msn-small'							).to(_lowerCamelCase							)

									UpperCamelCase_: Any   				=    self.default_image_processor
									UpperCamelCase_: List[Any]   				=    prepare_img()
									UpperCamelCase_: int   				=    image_processor(images=_lowerCamelCase  ,      return_tensors='pt'							).to(_lowerCamelCase							)

									# forward pass
									with torch.no_grad():
											UpperCamelCase_: Union[str, Any]   				=    model(**_lowerCamelCase							)

									# verify the logits
									UpperCamelCase_: str   				=    torch.Size((1, 1_0_0_0)							)
									self.assertEqual(outputs.logits.shape  ,      _lowerCamelCase							)

									UpperCamelCase_: str   				=    torch.tensor([-0.0_8_0_3, -0.4_4_5_4, -0.2_3_7_5]							).to(_lowerCamelCase							)

									self.assertTrue(torch.allclose(outputs.logits[0, :3]  ,      _lowerCamelCase  ,      atol=1e-4							)							)







import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / 'utils'))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
   from test_module.custom_tokenization_fast import CustomTokenizerFast




class     _lowerCAmelCase(	unittest.TestCase ):





       """simple docstring"""







       def       _a   ( self							):
         # A mock response for an HTTP head request to emulate server down
         UpperCamelCase_: Any   				=    mock.Mock()
         UpperCamelCase_: Dict   				=    5_0_0
         UpperCamelCase_: Any   				=    {}
         UpperCamelCase_: Tuple   				=    HTTPError
         UpperCamelCase_: List[str]   				=    {}

         # Download this model to make sure it's in the cache.
         UpperCamelCase_: int   				=    BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'							)

         # Under the mock environment we get a 500 error when trying to reach the tokenizer.
         with mock.patch('requests.Session.request'  ,      return_value=_lowerCamelCase							) as mock_head:
           UpperCamelCase_: Optional[int]   				=    BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'							)
           # This check we did call the fake head request
           mock_head.assert_called()





       @require_tokenizers
       def       _a   ( self							):
         # A mock response for an HTTP head request to emulate server down
         UpperCamelCase_: Union[str, Any]   				=    mock.Mock()
         UpperCamelCase_: Union[str, Any]   				=    5_0_0
         UpperCamelCase_: str   				=    {}
         UpperCamelCase_: List[str]   				=    HTTPError
         UpperCamelCase_: Optional[int]   				=    {}

         # Download this model to make sure it's in the cache.
         UpperCamelCase_: List[str]   				=    GPTaTokenizerFast.from_pretrained('gpt2'							)

         # Under the mock environment we get a 500 error when trying to reach the tokenizer.
         with mock.patch('requests.Session.request'  ,      return_value=_lowerCamelCase							) as mock_head:
           UpperCamelCase_: str   				=    GPTaTokenizerFast.from_pretrained('gpt2'							)
           # This check we did call the fake head request
           mock_head.assert_called()





       def       _a   ( self							):
         # This test is for deprecated behavior and can be removed in v5
         try:
           UpperCamelCase_: Optional[int]   				=    tempfile.mktemp()
           with open(_lowerCamelCase  ,      'wb'							) as f:
             http_get('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'  ,      _lowerCamelCase							)

           UpperCamelCase_: Tuple   				=    AlbertTokenizer.from_pretrained(_lowerCamelCase							)
         finally:
           os.remove(_lowerCamelCase							)

         # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
         # the current folder and have the right name.
         if os.path.isfile('tokenizer.json'							):
           # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
           return
         try:
           with open('tokenizer.json'  ,      'wb'							) as f:
             http_get('https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'  ,      _lowerCamelCase							)
           UpperCamelCase_: List[str]   				=    AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-gpt2'							)
           # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
           self.assertEqual(tokenizer.vocab_size  ,      1_0_0_0							)
           # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

         finally:
           os.remove('tokenizer.json'							)





       def       _a   ( self							):
         # This test is for deprecated behavior and can be removed in v5
         UpperCamelCase_: Any   				=    AlbertTokenizer.from_pretrained('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'							)







@is_staging_test
class     _lowerCAmelCase(	unittest.TestCase ):





       """simple docstring"""







       a    :		Dict									=['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''bla''', '''blou''']





       @classmethod
       def       _a   ( cls							):
         UpperCamelCase_: Optional[int]   				=    TOKEN
         HfFolder.save_token(_lowerCamelCase							)





       @classmethod
       def       _a   ( cls							):
         try:
           delete_repo(token=cls._token  ,      repo_id='test-tokenizer'							)
         except HTTPError:
           pass

         try:
           delete_repo(token=cls._token  ,      repo_id='valid_org/test-tokenizer-org'							)
         except HTTPError:
           pass

         try:
           delete_repo(token=cls._token  ,      repo_id='test-dynamic-tokenizer'							)
         except HTTPError:
           pass





       def       _a   ( self							):
         with tempfile.TemporaryDirectory() as tmp_dir:
           UpperCamelCase_: List[Any]   				=    os.path.join(_lowerCamelCase  ,      'vocab.txt'							)
           with open(_lowerCamelCase  ,      'w'  ,      encoding='utf-8'							) as vocab_writer:
             vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]							)							)
           UpperCamelCase_: int   				=    BertTokenizer(_lowerCamelCase							)

         tokenizer.push_to_hub('test-tokenizer'  ,      use_auth_token=self._token							)
         UpperCamelCase_: Union[str, Any]   				=    BertTokenizer.from_pretrained(f'''{USER}/test-tokenizer'''							)
         self.assertDictEqual(new_tokenizer.vocab  ,      tokenizer.vocab							)

         # Reset repo
         delete_repo(token=self._token  ,      repo_id='test-tokenizer'							)

         # Push to hub via save_pretrained
         with tempfile.TemporaryDirectory() as tmp_dir:
           tokenizer.save_pretrained(_lowerCamelCase  ,      repo_id='test-tokenizer'  ,      push_to_hub=_lowerCamelCase  ,      use_auth_token=self._token							)

         UpperCamelCase_: List[str]   				=    BertTokenizer.from_pretrained(f'''{USER}/test-tokenizer'''							)
         self.assertDictEqual(new_tokenizer.vocab  ,      tokenizer.vocab							)





       def       _a   ( self							):
         with tempfile.TemporaryDirectory() as tmp_dir:
           UpperCamelCase_: Optional[Any]   				=    os.path.join(_lowerCamelCase  ,      'vocab.txt'							)
           with open(_lowerCamelCase  ,      'w'  ,      encoding='utf-8'							) as vocab_writer:
             vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]							)							)
           UpperCamelCase_: Union[str, Any]   				=    BertTokenizer(_lowerCamelCase							)

         tokenizer.push_to_hub('valid_org/test-tokenizer-org'  ,      use_auth_token=self._token							)
         UpperCamelCase_: Dict   				=    BertTokenizer.from_pretrained('valid_org/test-tokenizer-org'							)
         self.assertDictEqual(new_tokenizer.vocab  ,      tokenizer.vocab							)

         # Reset repo
         delete_repo(token=self._token  ,      repo_id='valid_org/test-tokenizer-org'							)

         # Push to hub via save_pretrained
         with tempfile.TemporaryDirectory() as tmp_dir:
           tokenizer.save_pretrained(
               _lowerCamelCase  ,      repo_id='valid_org/test-tokenizer-org'  ,      push_to_hub=_lowerCamelCase  ,      use_auth_token=self._token							)

         UpperCamelCase_: Optional[Any]   				=    BertTokenizer.from_pretrained('valid_org/test-tokenizer-org'							)
         self.assertDictEqual(new_tokenizer.vocab  ,      tokenizer.vocab							)





       @require_tokenizers
       def       _a   ( self							):
         CustomTokenizer.register_for_auto_class()
         with tempfile.TemporaryDirectory() as tmp_dir:
           UpperCamelCase_: Optional[Any]   				=    os.path.join(_lowerCamelCase  ,      'vocab.txt'							)
           with open(_lowerCamelCase  ,      'w'  ,      encoding='utf-8'							) as vocab_writer:
             vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]							)							)
           UpperCamelCase_: Optional[int]   				=    CustomTokenizer(_lowerCamelCase							)

         # No fast custom tokenizer
         tokenizer.push_to_hub('test-dynamic-tokenizer'  ,      use_auth_token=self._token							)

         UpperCamelCase_: str   				=    AutoTokenizer.from_pretrained(f'''{USER}/test-dynamic-tokenizer'''  ,      trust_remote_code=_lowerCamelCase							)
         # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
         self.assertEqual(tokenizer.__class__.__name__  ,      'CustomTokenizer'							)

         # Fast and slow custom tokenizer
         CustomTokenizerFast.register_for_auto_class()
         with tempfile.TemporaryDirectory() as tmp_dir:
           UpperCamelCase_: Optional[Any]   				=    os.path.join(_lowerCamelCase  ,      'vocab.txt'							)
           with open(_lowerCamelCase  ,      'w'  ,      encoding='utf-8'							) as vocab_writer:
             vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]							)							)

           UpperCamelCase_: Dict   				=    BertTokenizerFast.from_pretrained(_lowerCamelCase							)
           bert_tokenizer.save_pretrained(_lowerCamelCase							)
           UpperCamelCase_: List[str]   				=    CustomTokenizerFast.from_pretrained(_lowerCamelCase							)

         tokenizer.push_to_hub('test-dynamic-tokenizer'  ,      use_auth_token=self._token							)

         UpperCamelCase_: Tuple   				=    AutoTokenizer.from_pretrained(f'''{USER}/test-dynamic-tokenizer'''  ,      trust_remote_code=_lowerCamelCase							)
         # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
         self.assertEqual(tokenizer.__class__.__name__  ,      'CustomTokenizerFast'							)
         UpperCamelCase_: int   				=    AutoTokenizer.from_pretrained(
             f'''{USER}/test-dynamic-tokenizer'''  ,      use_fast=_lowerCamelCase  ,      trust_remote_code=_lowerCamelCase							)
         # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
         self.assertEqual(tokenizer.__class__.__name__  ,      'CustomTokenizer'							)







class     _lowerCAmelCase(	unittest.TestCase ):





       """simple docstring"""







       def       _a   ( self							):
         UpperCamelCase_: Dict   				=    Trie()
         trie.add('Hello 友達'							)
         self.assertEqual(trie.data  ,      {'H': {'e': {'l': {'l': {'o': {' ': {'友': {'達': {'': 1}}}}}}}}}							)
         trie.add('Hello'							)
         trie.data
         self.assertEqual(trie.data  ,      {'H': {'e': {'l': {'l': {'o': {'': 1, ' ': {'友': {'達': {'': 1}}}}}}}}}							)





       def       _a   ( self							):
         UpperCamelCase_: Optional[int]   				=    Trie()
         self.assertEqual(trie.split('[CLS] This is a extra_id_100'							)  ,      ['[CLS] This is a extra_id_100']							)
         trie.add('[CLS]'							)
         trie.add('extra_id_1'							)
         trie.add('extra_id_100'							)
         self.assertEqual(trie.split('[CLS] This is a extra_id_100'							)  ,      ['[CLS]', ' This is a ', 'extra_id_100']							)





       def       _a   ( self							):
         UpperCamelCase_: int   				=    Trie()
         trie.add('A'							)
         self.assertEqual(trie.split('ABC'							)  ,      ['A', 'BC']							)
         self.assertEqual(trie.split('BCA'							)  ,      ['BC', 'A']							)





       def       _a   ( self							):
         UpperCamelCase_: str   				=    Trie()
         trie.add('TOKEN]'							)
         trie.add('[SPECIAL_TOKEN]'							)
         self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]'							)  ,      ['This is something ', '[SPECIAL_TOKEN]']							)





       def       _a   ( self							):
         UpperCamelCase_: Union[str, Any]   				=    Trie()
         trie.add('A'							)
         trie.add('P'							)
         trie.add('[SPECIAL_TOKEN]'							)
         self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]'							)  ,      ['This is something ', '[SPECIAL_TOKEN]']							)





       def       _a   ( self							):
         UpperCamelCase_: List[str]   				=    Trie()
         trie.add('AB'							)
         trie.add('B'							)
         trie.add('C'							)
         self.assertEqual(trie.split('ABC'							)  ,      ['AB', 'C']							)





       def       _a   ( self							):
         UpperCamelCase_: List[str]   				=    Trie()
         trie.add('ABC'							)
         trie.add('B'							)
         trie.add('CD'							)
         self.assertEqual(trie.split('ABCD'							)  ,      ['ABC', 'D']							)





       def       _a   ( self							):
         # Even if the offsets are wrong, we necessarily output correct string
         # parts.
         UpperCamelCase_: Union[str, Any]   				=    Trie()
         UpperCamelCase_: Any   				=    trie.cut_text('ABC'  ,      [0, 0, 2, 1, 2, 3]							)
         self.assertEqual(_lowerCamelCase  ,      ['AB', 'C']							)