





'''simple docstring'''


import inspect
import unittest

from transformers import ConvNextVaConfig
from transformers.models.auto import get_values
from transformers.models.auto.modeling_auto import MODEL_FOR_BACKBONE_MAPPING_NAMES, MODEL_MAPPING_NAMES
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
    import torch

    from transformers import ConvNextVaBackbone, ConvNextVaForImageClassification, ConvNextVaModel
    from transformers.models.convnextva.modeling_convnextva import CONVNEXTV2_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
    from PIL import Image

    from transformers import AutoImageProcessor






class     lowerCAmelCase__ :

       '''simple docstring'''

       def __init__( self					:				List[Any]       ,     a__					:				Optional[Any]       ,     a__					:				Any=13       ,     a__					:				str=32       ,     a__					:				Optional[int]=3       ,     a__					:				Tuple=4       ,     a__					:				Any=[10, 20, 30, 40]       ,     a__					:				Any=[2, 2, 3, 2]       ,     a__					:				List[Any]=True       ,     a__					:				List[str]=True       ,     a__					:				Union[str, Any]=37       ,     a__					:				Tuple="gelu"       ,     a__					:				Any=10       ,     a__					:				List[str]=0.02       ,     a__					:				List[Any]=["stage2", "stage3", "stage4"]       ,     a__					:				Any=[2, 3, 4]       ,     a__					:				int=None       ,     ):
              UpperCAmelCase  =						parent
              UpperCAmelCase  =						batch_size
              UpperCAmelCase  =						image_size
              UpperCAmelCase  =						num_channels
              UpperCAmelCase  =						num_stages
              UpperCAmelCase  =						hidden_sizes
              UpperCAmelCase  =						depths
              UpperCAmelCase  =						is_training
              UpperCAmelCase  =						use_labels
              UpperCAmelCase  =						intermediate_size
              UpperCAmelCase  =						hidden_act
              UpperCAmelCase  =						num_labels
              UpperCAmelCase  =						initializer_range
              UpperCAmelCase  =						out_features
              UpperCAmelCase  =						out_indices
              UpperCAmelCase  =						scope

       def 			__snake_case      ( self					:				Dict						):
              UpperCAmelCase  =						floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]						)

              UpperCAmelCase  =						None
              if self.use_labels:
                     UpperCAmelCase  =						ids_tensor([self.batch_size]       ,     self.num_labels						)

              UpperCAmelCase  =						self.get_config()

              return config, pixel_values, labels

       def 			__snake_case      ( self					:				Dict						):
              return ConvNextVaConfig(
                  num_channels=self.num_channels       ,     hidden_sizes=self.hidden_sizes       ,     depths=self.depths       ,     num_stages=self.num_stages       ,     hidden_act=self.hidden_act       ,     is_decoder=a__       ,     initializer_range=self.initializer_range       ,     out_features=self.out_features       ,     out_indices=self.out_indices       ,     num_labels=self.num_labels       ,     )

       def 			__snake_case      ( self					:				Dict       ,     a__					:				Tuple       ,     a__					:				Union[str, Any]       ,     a__					:				List[Any]						):
              UpperCAmelCase  =						ConvNextVaModel(config=a__						)
              model.to(a__						)
              model.eval()
              UpperCAmelCase  =						model(a__						)
              # expected last hidden states: B, C, H // 32, W // 32
              self.parent.assertEqual(
                  result.last_hidden_state.shape       ,     (self.batch_size, self.hidden_sizes[-1], self.image_size // 32, self.image_size // 32)       ,     )

       def 			__snake_case      ( self					:				Dict       ,     a__					:				Any       ,     a__					:				Dict       ,     a__					:				List[Any]						):
              UpperCAmelCase  =						ConvNextVaForImageClassification(a__						)
              model.to(a__						)
              model.eval()
              UpperCAmelCase  =						model(a__       ,     labels=a__						)
              self.parent.assertEqual(result.logits.shape       ,     (self.batch_size, self.num_labels)						)

       def 			__snake_case      ( self					:				Any       ,     a__					:				Optional[Any]       ,     a__					:				List[Any]       ,     a__					:				Dict						):
              UpperCAmelCase  =						ConvNextVaBackbone(config=a__						)
              model.to(a__						)
              model.eval()
              UpperCAmelCase  =						model(a__						)

              # verify hidden states
              self.parent.assertEqual(len(result.feature_maps						)       ,     len(config.out_features						)						)
              self.parent.assertListEqual(list(result.feature_maps[0].shape						)       ,     [self.batch_size, self.hidden_sizes[1], 4, 4]						)

              # verify channels
              self.parent.assertEqual(len(model.channels						)       ,     len(config.out_features						)						)
              self.parent.assertListEqual(model.channels       ,     config.hidden_sizes[1:]						)

              # verify backbone works with out_features=None
              UpperCAmelCase  =						None
              UpperCAmelCase  =						ConvNextVaBackbone(config=a__						)
              model.to(a__						)
              model.eval()
              UpperCAmelCase  =						model(a__						)

              # verify feature maps
              self.parent.assertEqual(len(result.feature_maps						)       ,     1						)
              self.parent.assertListEqual(list(result.feature_maps[0].shape						)       ,     [self.batch_size, self.hidden_sizes[-1], 1, 1]						)

              # verify channels
              self.parent.assertEqual(len(model.channels						)       ,     1						)
              self.parent.assertListEqual(model.channels       ,     [config.hidden_sizes[-1]]						)

       def 			__snake_case      ( self					:				Any						):
              UpperCAmelCase  =						self.prepare_config_and_inputs()
              UpperCAmelCase,					UpperCAmelCase,					UpperCAmelCase  =						config_and_inputs
              UpperCAmelCase  =						{'''pixel_values''': pixel_values}
              return config, inputs_dict



       def 			__snake_case      ( self					:				List[str]						):
              UpperCAmelCase  =						self.prepare_config_and_inputs()
              UpperCAmelCase,					UpperCAmelCase,					UpperCAmelCase  =						config_and_inputs
              UpperCAmelCase  =						{'''pixel_values''': pixel_values, '''labels''': labels}
              return config, inputs_dict






@require_torch
class     lowerCAmelCase__ (				UpperCAmelCase_      , UpperCAmelCase_      , unittest.TestCase    ):

       '''simple docstring'''
       _lowerCamelCase								=(
           (
               ConvNextVaModel,
               ConvNextVaForImageClassification,
               ConvNextVaBackbone,
           )
           if is_torch_available()
           else ()
       )
       _lowerCamelCase								=(
           {"feature-extraction": ConvNextVaModel, "image-classification": ConvNextVaForImageClassification}
           if is_torch_available()
           else {}
       )

       _lowerCamelCase								=False
       _lowerCamelCase								=False
       _lowerCamelCase								=False
       _lowerCamelCase								=False
       _lowerCamelCase								=False

       def 			__snake_case      ( self					:				List[Any]						):
              UpperCAmelCase  =						ConvNextVaModelTester(self						)
              UpperCAmelCase  =						ConfigTester(self       ,     config_class=a__       ,     has_text_modality=a__       ,     hidden_size=37						)

       def 			__snake_case      ( self					:				Tuple						):
              self.create_and_test_config_common_properties()
              self.config_tester.create_and_test_config_to_json_string()
              self.config_tester.create_and_test_config_to_json_file()
              self.config_tester.create_and_test_config_from_and_save_pretrained()
              self.config_tester.create_and_test_config_with_num_labels()
              self.config_tester.check_config_can_be_init_without_params()
              self.config_tester.check_config_arguments_init()

       def 			__snake_case      ( self					:				Optional[Any]						):
              return

       @unittest.skip(reason='''ConvNextV2 does not use inputs_embeds'''						)
       def 			__snake_case      ( self					:				List[str]						):
              pass

       @unittest.skip(reason='''ConvNextV2 does not support input and output embeddings'''						)
       def 			__snake_case      ( self					:				Union[str, Any]						):
              pass

       @unittest.skip(reason='''ConvNextV2 does not use feedforward chunking'''						)
       def 			__snake_case      ( self					:				str						):
              pass

       def 			__snake_case      ( self					:				Optional[Any]						):
              if not self.model_tester.is_training:
                     return

              for model_class in self.all_model_classes:
                     UpperCAmelCase,					UpperCAmelCase  =						self.model_tester.prepare_config_and_inputs_with_labels()
                     UpperCAmelCase  =						True

                     if model_class.__name__ in [
                         *get_values(a__						),
                         *get_values(a__						),
                     ]:
                            continue

                     UpperCAmelCase  =						model_class(a__						)
                     model.to(a__						)
                     model.train()
                     UpperCAmelCase  =						self._prepare_for_class(a__       ,     a__       ,     return_labels=a__						)
                     UpperCAmelCase  =						model(**a__						).loss
                     loss.backward()

       def 			__snake_case      ( self					:				Union[str, Any]						):
              if not self.model_tester.is_training:
                     return

              for model_class in self.all_model_classes:
                     UpperCAmelCase,					UpperCAmelCase  =						self.model_tester.prepare_config_and_inputs_with_labels()
                     UpperCAmelCase  =						False
                     UpperCAmelCase  =						True

                     if (
                         model_class.__name__
                         in [*get_values(a__						), *get_values(a__						)]
                         or not model_class.supports_gradient_checkpointing
                     ):
                            continue

                     UpperCAmelCase  =						model_class(a__						)
                     model.to(a__						)
                     model.gradient_checkpointing_enable()
                     model.train()
                     UpperCAmelCase  =						self._prepare_for_class(a__       ,     a__       ,     return_labels=a__						)
                     UpperCAmelCase  =						model(**a__						).loss
                     loss.backward()

       def 			__snake_case      ( self					:				str						):
              UpperCAmelCase,					UpperCAmelCase  =						self.model_tester.prepare_config_and_inputs_for_common()

              for model_class in self.all_model_classes:
                     UpperCAmelCase  =						model_class(a__						)
                     UpperCAmelCase  =						inspect.signature(model.forward						)
                     # signature.parameters is an OrderedDict => so arg_names order is deterministic
                     UpperCAmelCase  =						[*signature.parameters.keys()]

                     UpperCAmelCase  =						['''pixel_values''']
                     self.assertListEqual(arg_names[:1]       ,     a__						)

       def 			__snake_case      ( self					:				Any						):
              UpperCAmelCase  =						self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_model(*a__						)

       def 			__snake_case      ( self					:				Any						):
              def check_hidden_states_output(a__					:				Dict       ,     a__					:				str       ,     a__					:				Dict						):
                     UpperCAmelCase  =						model_class(a__						)
                     model.to(a__						)
                     model.eval()

                     with torch.no_grad():
                            UpperCAmelCase  =						model(**self._prepare_for_class(a__       ,     a__						)						)

                     UpperCAmelCase  =						outputs.encoder_hidden_states if config.is_encoder_decoder else outputs.hidden_states

                     UpperCAmelCase  =						self.model_tester.num_stages
                     self.assertEqual(len(a__						)       ,     expected_num_stages + 1						)

                     # ConvNextV2's feature maps are of shape (batch_size, num_channels, height, width)
                     self.assertListEqual(
                         list(hidden_states[0].shape[-2:]						)       ,     [self.model_tester.image_size // 4, self.model_tester.image_size // 4]       ,     )

              UpperCAmelCase,					UpperCAmelCase  =						self.model_tester.prepare_config_and_inputs_for_common()

              for model_class in self.all_model_classes:
                     UpperCAmelCase  =						True
                     check_hidden_states_output(a__       ,     a__       ,     a__						)

                     # check that output_hidden_states also work using config
                     del inputs_dict["output_hidden_states"]
                     UpperCAmelCase  =						True

                     check_hidden_states_output(a__       ,     a__       ,     a__						)

       def 			__snake_case      ( self					:				Union[str, Any]						):
              UpperCAmelCase  =						self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_for_image_classification(*a__						)



       @slow
       def 			__snake_case      ( self					:				Tuple						):
              for model_name in CONVNEXTV2_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
                     UpperCAmelCase  =						ConvNextVaModel.from_pretrained(a__						)
                     self.assertIsNotNone(a__						)


def       __snake_case		( )							->     int:

       """simple docstring"""


       UpperCAmelCase  =						Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png''' )
       return image






@require_torch
@require_vision
class     lowerCAmelCase__ (				unittest.TestCase    ):

       '''simple docstring'''

       @cached_property
       def 			__snake_case      ( self					:				Dict						):
              return AutoImageProcessor.from_pretrained('''facebook/convnextv2-tiny-1k-224'''						) if is_vision_available() else None



       @slow
       def 			__snake_case      ( self					:				Dict						):
              UpperCAmelCase  =						ConvNextVaForImageClassification.from_pretrained('''facebook/convnextv2-tiny-1k-224'''						).to(a__						)

              UpperCAmelCase  =						self.default_image_processor
              UpperCAmelCase  =						prepare_img()
              UpperCAmelCase  =						preprocessor(images=a__       ,     return_tensors='''pt'''						).to(a__						)

              # forward pass
              with torch.no_grad():
                     UpperCAmelCase  =						model(**a__						)

              # verify the logits
              UpperCAmelCase  =						torch.Size((1, 1000)						)
              self.assertEqual(outputs.logits.shape       ,     a__						)

              UpperCAmelCase  =						torch.tensor([0.9_996, 0.1_966, -0.4_386]						).to(a__						)
              self.assertTrue(torch.allclose(outputs.logits[0, :3]       ,     a__       ,     atol=1e-4						)						)






import collections
import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


SCREAMING_SNAKE_CASE							    =							logging.get_logger(__name__)

SCREAMING_SNAKE_CASE							    =							'▁'

SCREAMING_SNAKE_CASE							    =							{'vocab_file': 'prophetnet.tokenizer'}

SCREAMING_SNAKE_CASE							    =							{
    'vocab_file': {
        'microsoft/xprophetnet-large-wiki100-cased': (
            'https://huggingface.co/microsoft/xprophetnet-large-wiki100-cased/resolve/main/prophetnet.tokenizer'
        ),
    }
}

SCREAMING_SNAKE_CASE							    =							{
    'microsoft/xprophetnet-large-wiki100-cased': {'do_lower_case': False},
}

SCREAMING_SNAKE_CASE							    =							{
    'microsoft/xprophetnet-large-wiki100-cased': 5_1_2,
}




def  a   (lowerCAmelCase__   ):
   __a					=  collections.OrderedDict()
   with open(lowerCAmelCase__		,				"""r"""		,				encoding="""utf-8"""   ) as reader:
      __a					=  reader.readlines()
   for index, token in enumerate(lowerCAmelCase__   ):
      __a					=  token.rstrip("""\n"""   )
      __a					=  index
   return vocab


class     __UpperCAmelCase							(       __A							):


     """simple docstring"""
     _lowerCamelCase   =      VOCAB_FILES_NAMES
     _lowerCamelCase   =      PRETRAINED_VOCAB_FILES_MAP
     _lowerCamelCase   =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     _lowerCamelCase   =      ["""input_ids""", """attention_mask"""]


     def __init__(	self  ,       __A  ,       __A="[SEP]"  ,       __A="[SEP]"  ,       __A="[SEP]"  ,       __A="[UNK]"  ,       __A="[PAD]"  ,       __A="[CLS]"  ,       __A="[MASK]"  ,       __A = None  ,       **__A  ,       ):
        __a					=  {} if sp_model_kwargs is None else sp_model_kwargs

        super().__init__(
            bos_token=__A  ,       eos_token=__A  ,       sep_token=__A  ,       unk_token=__A  ,       pad_token=__A  ,       cls_token=__A  ,       mask_token=__A  ,       sp_model_kwargs=self.sp_model_kwargs  ,       **__A  ,       )

        try:
           import sentencepiece as spm
        except ImportError:
           logger.warning(
               """You need to install SentencePiece to use XLMRobertaTokenizer: https://github.com/google/sentencepiece"""
               """ pip install sentencepiece"""    )
           raise

        __a					=  spm.SentencePieceProcessor(**self.sp_model_kwargs    )
        self.sp_model.Load(str(__A    )    )
        __a					=  vocab_file

        # Original fairseq vocab and spm vocab must be "aligned":
        # Vocab    |    0    |    1    |   2    |    3    |  4  |  5  |  6  |   7   |   8   |  9
        # -------- | ------- | ------- | ------ | ------- | --- | --- | --- | ----- | ----- | ----
        # fairseq  | '<s>'   | '<pad>' | '</s>' | '<unk>' | ',' | '.' | '▁' | 's'   | '▁de' | '-'
        # spm      | '<unk>' | '<s>'   | '</s>' | ','     | '.' | '▁' | 's' | '▁de' | '-'   | '▁a'

        # put special tokens and [unused] tokens into the vocab
        __a					=  {"""[PAD]""": 0, """[CLS]""": 1, """[SEP]""": 2, """[UNK]""": 3, """[MASK]""": 4}

        for i in range(10    ):
           __a					=  f'''[unused{i}]'''
           __a					=  5 + i

        # The first "real" token "," has position 15 in the embedding vocab and position 3 in the spm vocab
        __a					=  12
        __a					=  {v: k for k, v in self.fairseq_tokens_to_ids.items()}
        for k in self.fairseq_tokens_to_ids.keys():
           self.unique_no_split_tokens.append(__A    )


     def __getstate__(	self    ):
        __a					=  self.__dict__.copy()
        __a					=  None
        return state


     def __setstate__(	self  ,       __A    ):
        __a					=  d
        try:
           import sentencepiece as spm
        except ImportError:
           logger.warning(
               """You need to install SentencePiece to use XLMRobertaTokenizer: https://github.com/google/sentencepiece"""
               """ pip install sentencepiece"""    )
           raise

        # for backward compatibility
        if not hasattr(self  ,       """sp_model_kwargs"""    ):
           __a					=  {}

        __a					=  spm.SentencePieceProcessor(**self.sp_model_kwargs    )
        self.sp_model.Load(self.vocab_file    )


     def 	snake_case_  (	self  ,       __A  ,       __A = None  ,       __A = False    ):

        if already_has_special_tokens:
           return super().get_special_tokens_mask(
               token_ids_a=__A  ,       token_ids_a=__A  ,       already_has_special_tokens=__A    )

        if token_ids_a is None:
           return ([0] * len(__A    )) + [1]
        return ([0] * len(__A    )) + [1] + ([0] * len(__A    )) + [1]


     def 	snake_case_  (	self  ,       __A  ,       __A = None    ):
        __a					=  [self.sep_token_id]

        if token_ids_a is None:
           return len(token_ids_a + sep    ) * [0]
        return len(token_ids_a + sep + sep + token_ids_a + sep    ) * [0]


     @property
     def 	snake_case_  (	self    ):
        return len(self.sp_model    ) + self.fairseq_offset


     def 	snake_case_  (	self    ):
        __a					=  {self.convert_ids_to_tokens(__A    ): i for i in range(self.vocab_size    )}
        vocab.update(self.added_tokens_encoder    )
        return vocab


     def 	snake_case_  (	self  ,       __A    ):
        return self.sp_model.encode(__A  ,       out_type=__A    )


     def 	snake_case_  (	self  ,       __A    ):
        if token in self.fairseq_tokens_to_ids:
           return self.fairseq_tokens_to_ids[token]
        __a					=  self.sp_model.PieceToId(__A    )

        # Need to return unknown token if the SP model returned 0
        return spm_id + self.fairseq_offset if spm_id else self.unk_token_id


     def 	snake_case_  (	self  ,       __A    ):
        if index in self.fairseq_ids_to_tokens:
           return self.fairseq_ids_to_tokens[index]
        return self.sp_model.IdToPiece(index - self.fairseq_offset    )


     def 	snake_case_  (	self  ,       __A    ):
        __a					=  """""".join(__A    ).replace(__A  ,       """ """    ).strip()
        return out_string


     def 	snake_case_  (	self  ,       __A  ,       __A = None    ):
        if not os.path.isdir(__A    ):
           logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''    )
           return
        __a					=  os.path.join(
            __A  ,       (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]    )

        if os.path.abspath(self.vocab_file    ) != os.path.abspath(__A    ) and os.path.isfile(self.vocab_file    ):
           copyfile(self.vocab_file  ,       __A    )
        elif not os.path.isfile(self.vocab_file    ):
           with open(__A  ,       """wb"""    ) as fi:
              __a					=  self.sp_model.serialized_model_proto()
              fi.write(__A    )

        return (out_vocab_file,)


     def 	snake_case_  (	self  ,       __A  ,       __A = None    ):

        if token_ids_a is None:
           return token_ids_a + [self.sep_token_id]
        __a					=  [self.sep_token_id]
        return token_ids_a + sep + token_ids_a + sep



