






'''simple docstring'''





import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from requests.exceptions import HTTPError

from transformers import AutoImageProcessor, ViTImageProcessor
from transformers.testing_utils import TOKEN, USER, get_tests_dir, is_staging_test


sys.path.append(str(Path(__file__).parent.parent / """utils"""))

from test_module.custom_image_processing import CustomImageProcessor  # noqa E402


__A							:							Optional[Any]  							=		get_tests_dir("""fixtures""")
class 				lowercase       (							unittest.TestCase      ):


  '''simple docstring'''

  def    a__				(					self     :     Optional[Any]     )							->							Union[str, Any]:

   '''simple docstring'''




   lowerCamelCase__  =     mock.Mock()
   lowerCamelCase__  =     500
   lowerCamelCase__  =     {}
   lowerCamelCase__  =     HTTPError
   lowerCamelCase__  =     {}

   # Download this model to make sure it's in the cache.
   lowerCamelCase__  =     ViTImageProcessor.from_pretrained("hf-internal-testing/tiny-random-vit"     )
   # Under the mock environment we get a 500 error when trying to reach the model.
   with mock.patch("requests.Session.request"				,     return_value=__lowerCamelCase     ) as mock_head:
    lowerCamelCase__  =     ViTImageProcessor.from_pretrained("hf-internal-testing/tiny-random-vit"     )
    # This check we did call the fake head request
    mock_head.assert_called()

  def    a__				(					self     :     Union[str, Any]     )							->							List[Any]:

   '''simple docstring'''




   lowerCamelCase__  =     ViTImageProcessor.from_pretrained(
       "https://huggingface.co/hf-internal-testing/tiny-random-vit/resolve/main/preprocessor_config.json"     )




  def    a__				(					self     :     Optional[Any]     )							->							List[str]:

   '''simple docstring'''




   with self.assertRaises(__lowerCamelCase     ):
    # config is in subfolder, the following should not work without specifying the subfolder
    lowerCamelCase__  =     AutoImageProcessor.from_pretrained("hf-internal-testing/stable-diffusion-all-variants"     )

   lowerCamelCase__  =     AutoImageProcessor.from_pretrained(
       "hf-internal-testing/stable-diffusion-all-variants"				,     subfolder="feature_extractor"     )

   self.assertIsNotNone(__lowerCamelCase     )
@is_staging_test
class 				lowercase       (							unittest.TestCase      ):


  '''simple docstring'''

  @classmethod
  def    a__				(					cls     :     List[str]     )							->							Any:

   '''simple docstring'''




   lowerCamelCase__  =     TOKEN
   HfFolder.save_token(__lowerCamelCase     )

  @classmethod
  def    a__				(					cls     :     Optional[Any]     )							->							Dict:

   '''simple docstring'''




   try:
    delete_repo(token=cls._token				,     repo_id="test-image-processor"     )
   except HTTPError:
    pass

   try:
    delete_repo(token=cls._token				,     repo_id="valid_org/test-image-processor-org"     )
   except HTTPError:
    pass

   try:
    delete_repo(token=cls._token				,     repo_id="test-dynamic-image-processor"     )
   except HTTPError:
    pass

  def    a__				(					self     :     Dict     )							->							Optional[Any]:

   '''simple docstring'''




   lowerCamelCase__  =     ViTImageProcessor.from_pretrained(__lowerCamelCase     )
   image_processor.push_to_hub("test-image-processor"				,     use_auth_token=self._token     )

   lowerCamelCase__  =     ViTImageProcessor.from_pretrained(f'''{USER}/test-image-processor'''     )
   for k, v in image_processor.__dict__.items():
    self.assertEqual(__lowerCamelCase				,     getattr(__lowerCamelCase				,     __lowerCamelCase     )     )

   # Reset repo
   delete_repo(token=self._token				,     repo_id="test-image-processor"     )

   # Push to hub via save_pretrained
   with tempfile.TemporaryDirectory() as tmp_dir:
    image_processor.save_pretrained(
        __lowerCamelCase				,     repo_id="test-image-processor"				,     push_to_hub=__lowerCamelCase				,     use_auth_token=self._token     )

   lowerCamelCase__  =     ViTImageProcessor.from_pretrained(f'''{USER}/test-image-processor'''     )
   for k, v in image_processor.__dict__.items():
    self.assertEqual(__lowerCamelCase				,     getattr(__lowerCamelCase				,     __lowerCamelCase     )     )

  def    a__				(					self     :     int     )							->							List[str]:

   '''simple docstring'''




   lowerCamelCase__  =     ViTImageProcessor.from_pretrained(__lowerCamelCase     )
   image_processor.push_to_hub("valid_org/test-image-processor"				,     use_auth_token=self._token     )

   lowerCamelCase__  =     ViTImageProcessor.from_pretrained("valid_org/test-image-processor"     )
   for k, v in image_processor.__dict__.items():
    self.assertEqual(__lowerCamelCase				,     getattr(__lowerCamelCase				,     __lowerCamelCase     )     )

   # Reset repo
   delete_repo(token=self._token				,     repo_id="valid_org/test-image-processor"     )

   # Push to hub via save_pretrained
   with tempfile.TemporaryDirectory() as tmp_dir:
    image_processor.save_pretrained(
        __lowerCamelCase				,     repo_id="valid_org/test-image-processor-org"				,     push_to_hub=__lowerCamelCase				,     use_auth_token=self._token     )

   lowerCamelCase__  =     ViTImageProcessor.from_pretrained("valid_org/test-image-processor-org"     )
   for k, v in image_processor.__dict__.items():
    self.assertEqual(__lowerCamelCase				,     getattr(__lowerCamelCase				,     __lowerCamelCase     )     )




  def    a__				(					self     :     Dict     )							->							str:

   '''simple docstring'''




   CustomImageProcessor.register_for_auto_class()
   lowerCamelCase__  =     CustomImageProcessor.from_pretrained(__lowerCamelCase     )

   image_processor.push_to_hub("test-dynamic-image-processor"				,     use_auth_token=self._token     )

   # This has added the proper auto_map field to the config
   self.assertDictEqual(
       image_processor.auto_map				,     {"AutoImageProcessor": "custom_image_processing.CustomImageProcessor"}				,     )

   lowerCamelCase__  =     AutoImageProcessor.from_pretrained(
       f'''{USER}/test-dynamic-image-processor'''				,     trust_remote_code=__lowerCamelCase     )
   # Can't make an isinstance check because the new_image_processor is from the CustomImageProcessor class of a dynamic module
   self.assertEqual(new_image_processor.__class__.__name__				,     "CustomImageProcessor"     )









'''simple docstring'''





import tensorflow as tf

from ...tf_utils import shape_list
class 				lowercase       (							tf.keras.layers.Layer      ):


    '''simple docstring'''

    def __init__(					self     :     Union[str, Any]				,     __lowerCamelCase     :     str				,     __lowerCamelCase     :     int				,     __lowerCamelCase     :     Dict				,     __lowerCamelCase     :     Union[str, Any]				,     __lowerCamelCase     :     int=1				,     __lowerCamelCase     :     Tuple=False				,     **__lowerCamelCase     :     Dict     )							->							str:

     '''simple docstring'''




     super().__init__(**__lowerCamelCase     )

     lowerCamelCase__  =     vocab_size
     lowerCamelCase__  =     d_embed
     lowerCamelCase__  =     d_proj

     lowerCamelCase__  =     cutoffs + [vocab_size]
     lowerCamelCase__  =     [0] + self.cutoffs
     lowerCamelCase__  =     div_val

     lowerCamelCase__  =     self.cutoffs[0]
     lowerCamelCase__  =     len(self.cutoffs     ) - 1
     lowerCamelCase__  =     self.shortlist_size + self.n_clusters
     lowerCamelCase__  =     keep_order

     lowerCamelCase__  =     []
     lowerCamelCase__  =     []

    def    a__				(					self     :     Optional[int]				,     __lowerCamelCase     :     str     )							->							List[str]:

     '''simple docstring'''




     if self.n_clusters > 0:
      lowerCamelCase__  =     self.add_weight(
          shape=(self.n_clusters, self.d_embed)				,     initializer="zeros"				,     trainable=__lowerCamelCase				,     name="cluster_weight"     )
      lowerCamelCase__  =     self.add_weight(
          shape=(self.n_clusters,)				,     initializer="zeros"				,     trainable=__lowerCamelCase				,     name="cluster_bias"     )

     if self.div_val == 1:
      for i in range(len(self.cutoffs     )     ):
       if self.d_proj != self.d_embed:
        lowerCamelCase__  =     self.add_weight(
            shape=(self.d_embed, self.d_proj)				,     initializer="zeros"				,     trainable=__lowerCamelCase				,     name=f'''out_projs_._{i}'''				,     )
        self.out_projs.append(__lowerCamelCase     )
       else:
        self.out_projs.append(__lowerCamelCase     )
       lowerCamelCase__  =     self.add_weight(
           shape=(self.vocab_size, self.d_embed)				,     initializer="zeros"				,     trainable=__lowerCamelCase				,     name=f'''out_layers_._{i}_._weight'''				,     )
       lowerCamelCase__  =     self.add_weight(
           shape=(self.vocab_size,)				,     initializer="zeros"				,     trainable=__lowerCamelCase				,     name=f'''out_layers_._{i}_._bias'''				,     )
       self.out_layers.append((weight, bias)     )
     else:
      for i in range(len(self.cutoffs     )     ):
       lowerCamelCase__  ,	lowerCamelCase__  =     self.cutoff_ends[i], self.cutoff_ends[i + 1]
       lowerCamelCase__  =     self.d_embed // (self.div_val**i)

       lowerCamelCase__  =     self.add_weight(
           shape=(d_emb_i, self.d_proj)				,     initializer="zeros"				,     trainable=__lowerCamelCase				,     name=f'''out_projs_._{i}'''     )
       self.out_projs.append(__lowerCamelCase     )
       lowerCamelCase__  =     self.add_weight(
           shape=(r_idx - l_idx, d_emb_i)				,     initializer="zeros"				,     trainable=__lowerCamelCase				,     name=f'''out_layers_._{i}_._weight'''				,     )
       lowerCamelCase__  =     self.add_weight(
           shape=(r_idx - l_idx,)				,     initializer="zeros"				,     trainable=__lowerCamelCase				,     name=f'''out_layers_._{i}_._bias'''				,     )
       self.out_layers.append((weight, bias)     )
     super().build(__lowerCamelCase     )

    @staticmethod
    def    a__				(					__lowerCamelCase     :     Union[str, Any]				,     __lowerCamelCase     :     Optional[Any]				,     __lowerCamelCase     :     Tuple				,     __lowerCamelCase     :     List[str]=None     )							->							str:

     '''simple docstring'''




     lowerCamelCase__  =     x
     if proj is not None:
      lowerCamelCase__  =     tf.einsum("ibd,ed->ibe"				,     __lowerCamelCase				,     __lowerCamelCase     )
     return tf.einsum("ibd,nd->ibn"				,     __lowerCamelCase				,     __lowerCamelCase     ) + b

    @staticmethod
    def    a__				(					__lowerCamelCase     :     List[str]				,     __lowerCamelCase     :     Union[str, Any]     )							->							Union[str, Any]:

     '''simple docstring'''




     lowerCamelCase__  =     shape_list(__lowerCamelCase     )
     lowerCamelCase__  =     tf.range(lp_size[0]				,     dtype=target.dtype     )
     lowerCamelCase__  =     tf.stack([r, target]				,     1     )
     return tf.gather_nd(__lowerCamelCase				,     __lowerCamelCase     )




    def    a__				(					self     :     Any				,     __lowerCamelCase     :     int				,     __lowerCamelCase     :     Tuple				,     __lowerCamelCase     :     str=True				,     __lowerCamelCase     :     Tuple=False     )							->							int:

     '''simple docstring'''




     lowerCamelCase__  =     0
     if self.n_clusters == 0:
      lowerCamelCase__  =     self._logit(__lowerCamelCase				,     self.out_layers[0][0]				,     self.out_layers[0][1]				,     self.out_projs[0]     )
      if target is not None:
       lowerCamelCase__  =     tf.nn.sparse_softmax_cross_entropy_with_logits(labels=__lowerCamelCase				,     logits=__lowerCamelCase     )
      lowerCamelCase__  =     tf.nn.log_softmax(__lowerCamelCase				,     axis=-1     )
     else:
      lowerCamelCase__  =     shape_list(__lowerCamelCase     )
      lowerCamelCase__  =     []
      lowerCamelCase__  =     tf.zeros(hidden_sizes[:2]     )
      for i in range(len(self.cutoffs     )     ):
       lowerCamelCase__  ,	lowerCamelCase__  =     self.cutoff_ends[i], self.cutoff_ends[i + 1]
       if target is not None:
        lowerCamelCase__  =     (target >= l_idx) & (target < r_idx)
        lowerCamelCase__  =     tf.where(__lowerCamelCase     )
        lowerCamelCase__  =     tf.boolean_mask(__lowerCamelCase				,     __lowerCamelCase     ) - l_idx

       if self.div_val == 1:
        lowerCamelCase__  =     self.out_layers[0][0][l_idx:r_idx]
        lowerCamelCase__  =     self.out_layers[0][1][l_idx:r_idx]
       else:
        lowerCamelCase__  =     self.out_layers[i][0]
        lowerCamelCase__  =     self.out_layers[i][1]

       if i == 0:
        lowerCamelCase__  =     tf.concat([cur_W, self.cluster_weight]				,     0     )
        lowerCamelCase__  =     tf.concat([cur_b, self.cluster_bias]				,     0     )

        lowerCamelCase__  =     self._logit(__lowerCamelCase				,     __lowerCamelCase				,     __lowerCamelCase				,     self.out_projs[0]     )
        lowerCamelCase__  =     tf.nn.log_softmax(__lowerCamelCase     )
        out.append(head_logprob[..., : self.cutoffs[0]]     )
        if target is not None:
         lowerCamelCase__  =     tf.boolean_mask(__lowerCamelCase				,     __lowerCamelCase     )
         lowerCamelCase__  =     self._gather_logprob(__lowerCamelCase				,     __lowerCamelCase     )
       else:
        lowerCamelCase__  =     self._logit(__lowerCamelCase				,     __lowerCamelCase				,     __lowerCamelCase				,     self.out_projs[i]     )
        lowerCamelCase__  =     tf.nn.log_softmax(__lowerCamelCase     )
        lowerCamelCase__  =     self.cutoffs[0] + i - 1  # No probability for the head cluster
        lowerCamelCase__  =     head_logprob[..., cluster_prob_idx, None] + tail_logprob
        out.append(__lowerCamelCase     )
        if target is not None:
         lowerCamelCase__  =     tf.boolean_mask(__lowerCamelCase				,     __lowerCamelCase     )
         lowerCamelCase__  =     tf.boolean_mask(__lowerCamelCase				,     __lowerCamelCase     )
         lowerCamelCase__  =     self._gather_logprob(__lowerCamelCase				,     __lowerCamelCase     )
         cur_logprob += cur_head_logprob[:, self.cutoff_ends[1] + i - 1]
       if target is not None:
        loss += tf.scatter_nd(__lowerCamelCase				,     -cur_logprob				,     shape_list(__lowerCamelCase     )     )
      lowerCamelCase__  =     tf.concat(__lowerCamelCase				,     axis=-1     )

     if target is not None:
      if return_mean:
       lowerCamelCase__  =     tf.reduce_mean(__lowerCamelCase     )
      # Add the training-time loss value to the layer using `self.add_loss()`.
      self.add_loss(__lowerCamelCase     )

      # Log the loss as a metric (we could log arbitrary metrics,
      # including different metrics for training and inference.
      self.add_metric(__lowerCamelCase				,     name=self.name				,     aggregation="mean" if return_mean else ""     )

     return out

