

import os
from pathlib import Path
from unittest.mock import patch

import pytest
import zstandard as zstd

from datasets.download.download_config import DownloadConfig
from datasets.utils.file_utils import (
    OfflineModeIsEnabled,
    cached_path,
    fsspec_get,
    fsspec_head,
    ftp_get,
    ftp_head,
    get_from_cache,
    http_get,
    http_head,
)


__UpperCAmelCase =				'\\n    Text data.\n    Second line of data.'

__UpperCAmelCase =				'file'
@pytest.fixture(scope="""session"""      )
def   lowerCAmelCase						(					__UpperCamelCase      ):
 '''simple docstring'''





 UpperCAmelCase__					:  List[str]					   =					tmp_path_factory.mktemp("""data"""      ) / (FILE_PATH + """.zstd""")
 UpperCAmelCase__					:  Any					   =					bytes(__UpperCamelCase							,    """utf-8"""      )
 with zstd.open(__UpperCamelCase							,    """wb"""      ) as f:
  f.write(__UpperCamelCase      )
 return path
@pytest.fixture
def   lowerCAmelCase						(					__UpperCamelCase      ):
 '''simple docstring'''





 with open(os.path.join(tmpfs.local_root_dir							,    __UpperCamelCase      )							,    """w"""      ) as f:
  f.write(__UpperCamelCase      )
 return FILE_PATH
@pytest.mark.parametrize("""compression_format"""							,    ["""gzip""", """xz""", """zstd"""]      )
def   lowerCAmelCase						(					__UpperCamelCase							,    __UpperCamelCase							,    __UpperCamelCase							,    __UpperCamelCase							,    __UpperCamelCase							,    __UpperCamelCase      ):
 '''simple docstring'''





 UpperCAmelCase__					:  Dict					   =					{"""gzip""": gz_file, """xz""": xz_file, """zstd""": zstd_path}
 UpperCAmelCase__					:  Optional[int]					   =					input_paths[compression_format]
 UpperCAmelCase__					:  Union[str, Any]					   =					tmp_path / """cache"""
 UpperCAmelCase__					:  List[Any]					   =					DownloadConfig(cache_dir=__UpperCamelCase							,    extract_compressed_file=__UpperCamelCase      )
 UpperCAmelCase__					:  Dict					   =					cached_path(__UpperCamelCase							,    download_config=__UpperCamelCase      )
 with open(__UpperCamelCase      ) as f:
  UpperCAmelCase__					:  Union[str, Any]					   =					f.read()
 with open(__UpperCamelCase      ) as f:
  UpperCAmelCase__					:  Tuple					   =					f.read()
 assert extracted_file_content == expected_file_content
@pytest.mark.parametrize("""default_extracted"""							,    [True, False]      )
@pytest.mark.parametrize("""default_cache_dir"""							,    [True, False]      )
def   lowerCAmelCase						(					__UpperCamelCase							,    __UpperCamelCase							,    __UpperCamelCase							,    __UpperCamelCase							,    __UpperCamelCase      ):
 '''simple docstring'''





 UpperCAmelCase__					:  Optional[int]					   =					"""custom_cache"""
 UpperCAmelCase__					:  Any					   =					"""custom_extracted_dir"""
 UpperCAmelCase__					:  Dict					   =					tmp_path / """custom_extracted_path"""
 if default_extracted:
  UpperCAmelCase__					:  Optional[int]					   =					("""downloads""" if default_cache_dir else custom_cache_dir, """extracted""")
 else:
  monkeypatch.setattr("""datasets.config.EXTRACTED_DATASETS_DIR"""							,    __UpperCamelCase      )
  monkeypatch.setattr("""datasets.config.EXTRACTED_DATASETS_PATH"""							,    str(__UpperCamelCase      )      )
  UpperCAmelCase__					:  Optional[Any]					   =					custom_extracted_path.parts[-2:] if default_cache_dir else (custom_cache_dir, custom_extracted_dir)

 UpperCAmelCase__					:  Tuple					   =					xz_file
 UpperCAmelCase__					:  List[Any]					   =					(
     DownloadConfig(extract_compressed_file=__UpperCamelCase      )
     if default_cache_dir
     else DownloadConfig(cache_dir=tmp_path / custom_cache_dir							,    extract_compressed_file=__UpperCamelCase      )
 )
 UpperCAmelCase__					:  Union[str, Any]					   =					cached_path(__UpperCamelCase							,    download_config=__UpperCamelCase      )
 assert Path(__UpperCamelCase      ).parent.parts[-2:] == expected
def   lowerCAmelCase						(					__UpperCamelCase      ):
 '''simple docstring'''





 UpperCAmelCase__					:  Union[str, Any]					   =					str(Path(__UpperCamelCase      ).resolve()      )
 assert cached_path(__UpperCamelCase      ) == text_file
 # relative path
 UpperCAmelCase__					:  int					   =					str(Path(__UpperCamelCase      ).resolve().relative_to(Path(os.getcwd()      )      )      )
 assert cached_path(__UpperCamelCase      ) == text_file
def   lowerCAmelCase						(					__UpperCamelCase      ):
 '''simple docstring'''





 UpperCAmelCase__					:  List[Any]					   =					str(tmp_path.resolve() / """__missing_file__.txt"""      )
 with pytest.raises(__UpperCamelCase      ):
  cached_path(__UpperCamelCase      )
 # relative path
 UpperCAmelCase__					:  str					   =					"""./__missing_file__.txt"""
 with pytest.raises(__UpperCamelCase      ):
  cached_path(__UpperCamelCase      )
def   lowerCAmelCase						(					__UpperCamelCase      ):
 '''simple docstring'''





 UpperCAmelCase__					:  str					   =					get_from_cache(F"tmp://{tmpfs_file}"      )
 with open(__UpperCamelCase      ) as f:
  UpperCAmelCase__					:  List[str]					   =					f.read()
 assert output_file_content == FILE_CONTENT
@patch("""datasets.config.HF_DATASETS_OFFLINE"""							,    __UpperCamelCase      )
def   lowerCAmelCase						(					):
 '''simple docstring'''





 with pytest.raises(__UpperCamelCase      ):
  cached_path("""https://huggingface.co"""      )
@patch("""datasets.config.HF_DATASETS_OFFLINE"""							,    __UpperCamelCase      )
def   lowerCAmelCase						(					__UpperCamelCase      ):
 '''simple docstring'''





 UpperCAmelCase__					:  Any					   =					tmp_path_factory.mktemp("""data"""      ) / """file.html"""
 with pytest.raises(__UpperCamelCase      ):
  http_get("""https://huggingface.co"""							,    temp_file=__UpperCamelCase      )
 with pytest.raises(__UpperCamelCase      ):
  http_head("""https://huggingface.co"""      )
@patch("""datasets.config.HF_DATASETS_OFFLINE"""							,    __UpperCamelCase      )
def   lowerCAmelCase						(					__UpperCamelCase      ):
 '''simple docstring'''





 UpperCAmelCase__					:  str					   =					tmp_path_factory.mktemp("""data"""      ) / """file.html"""
 with pytest.raises(__UpperCamelCase      ):
  ftp_get("""ftp://huggingface.co"""							,    temp_file=__UpperCamelCase      )
 with pytest.raises(__UpperCamelCase      ):
  ftp_head("""ftp://huggingface.co"""      )




@patch("""datasets.config.HF_DATASETS_OFFLINE"""							,    __UpperCamelCase      )
def   lowerCAmelCase						(					__UpperCamelCase      ):
 '''simple docstring'''





 UpperCAmelCase__					:  Dict					   =					tmp_path_factory.mktemp("""data"""      ) / """file.html"""
 with pytest.raises(__UpperCamelCase      ):
  fsspec_get("""s3://huggingface.co"""							,    temp_file=__UpperCamelCase      )
 with pytest.raises(__UpperCamelCase      ):
  fsspec_head("""s3://huggingface.co"""      )








"""simple docstring"""




from __future__ import annotations

import inspect
import unittest

import numpy as np

from transformers import DeiTConfig
from transformers.testing_utils import require_tf, require_vision, slow
from transformers.utils import cached_property, is_tf_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
      import tensorflow as tf

      from transformers import (
          TFDeiTForImageClassification,
          TFDeiTForImageClassificationWithTeacher,
          TFDeiTForMaskedImageModeling,
          TFDeiTModel,
      )
      from transformers.models.deit.modeling_tf_deit import TF_DEIT_PRETRAINED_MODEL_ARCHIVE_LIST


if is_vision_available():
      from PIL import Image

      from transformers import DeiTImageProcessor







class      __lowercase			:

  def __init__(  self		:			Optional[int]					,A		:			Optional[Any]					,A		:			int=13					,A		:			Any=30					,A		:			str=2					,A		:			Tuple=3					,A		:			Optional[Any]=True					,A		:			Union[str, Any]=True					,A		:			Dict=32					,A		:			Optional[int]=2					,A		:			Any=4					,A		:			str=37					,A		:			int="gelu"					,A		:			Dict=0.1					,A		:			str=0.1					,A		:			Optional[Any]=10					,A		:			List[str]=0.0_2					,A		:			Any=3					,A		:			Any=None					,A		:			int=2					,):






   '''simple docstring'''
   UpperCAmelCase__					:  List[Any]					   =					parent
   UpperCAmelCase__					:  str					   =					batch_size
   UpperCAmelCase__					:  Tuple					   =					image_size
   UpperCAmelCase__					:  int					   =					patch_size
   UpperCAmelCase__					:  Dict					   =					num_channels
   UpperCAmelCase__					:  List[Any]					   =					is_training
   UpperCAmelCase__					:  Optional[Any]					   =					use_labels
   UpperCAmelCase__					:  Tuple					   =					hidden_size
   UpperCAmelCase__					:  List[str]					   =					num_hidden_layers
   UpperCAmelCase__					:  Dict					   =					num_attention_heads
   UpperCAmelCase__					:  List[Any]					   =					intermediate_size
   UpperCAmelCase__					:  Optional[Any]					   =					hidden_act
   UpperCAmelCase__					:  Optional[Any]					   =					hidden_dropout_prob
   UpperCAmelCase__					:  str					   =					attention_probs_dropout_prob
   UpperCAmelCase__					:  Dict					   =					type_sequence_label_size
   UpperCAmelCase__					:  Optional[int]					   =					initializer_range
   UpperCAmelCase__					:  Tuple					   =					scope
   UpperCAmelCase__					:  str					   =					encoder_stride

   # in DeiT, the seq length equals the number of patches + 2 (we add 2 for the [CLS] and distilation tokens)
   UpperCAmelCase__					:  str					   =					(image_size // patch_size) ** 2
   UpperCAmelCase__					:  int					   =					num_patches + 2

  def    __lowercase       (  self		:			List[Any]    ):






   '''simple docstring'''
   UpperCAmelCase__					:  Union[str, Any]					   =					floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]    )

   UpperCAmelCase__					:  List[str]					   =					None
   if self.use_labels:
    UpperCAmelCase__					:  Optional[Any]					   =					ids_tensor([self.batch_size]					,self.type_sequence_label_size    )

   UpperCAmelCase__					:  int					   =					self.get_config()

   return config, pixel_values, labels

  def    __lowercase       (  self		:			Optional[Any]    ):






   '''simple docstring'''
   return DeiTConfig(
       image_size=self.image_size					,patch_size=self.patch_size					,num_channels=self.num_channels					,hidden_size=self.hidden_size					,num_hidden_layers=self.num_hidden_layers					,num_attention_heads=self.num_attention_heads					,intermediate_size=self.intermediate_size					,hidden_act=self.hidden_act					,hidden_dropout_prob=self.hidden_dropout_prob					,attention_probs_dropout_prob=self.attention_probs_dropout_prob					,is_decoder=A					,initializer_range=self.initializer_range					,encoder_stride=self.encoder_stride					,)

  def    __lowercase       (  self		:			Union[str, Any]					,A		:			Optional[int]					,A		:			Optional[Any]					,A		:			Optional[Any]    ):






   '''simple docstring'''
   UpperCAmelCase__					:  Dict					   =					TFDeiTModel(config=A    )
   UpperCAmelCase__					:  Dict					   =					model(A    )
   self.parent.assertEqual(result.last_hidden_state.shape					,(self.batch_size, self.seq_length, self.hidden_size)    )

  def    __lowercase       (  self		:			Optional[int]					,A		:			Union[str, Any]					,A		:			Optional[int]					,A		:			str    ):






   '''simple docstring'''
   UpperCAmelCase__					:  Dict					   =					TFDeiTForMaskedImageModeling(config=A    )
   UpperCAmelCase__					:  Any					   =					model(A    )
   self.parent.assertEqual(
       result.reconstruction.shape					,(self.batch_size, self.num_channels, self.image_size, self.image_size)    )

   # test greyscale images
   UpperCAmelCase__					:  Union[str, Any]					   =					1
   UpperCAmelCase__					:  Any					   =					TFDeiTForMaskedImageModeling(A    )

   UpperCAmelCase__					:  str					   =					floats_tensor([self.batch_size, 1, self.image_size, self.image_size]    )
   UpperCAmelCase__					:  str					   =					model(A    )
   self.parent.assertEqual(result.reconstruction.shape					,(self.batch_size, 1, self.image_size, self.image_size)    )

  def    __lowercase       (  self		:			Dict					,A		:			Any					,A		:			Optional[Any]					,A		:			List[Any]    ):






   '''simple docstring'''
   UpperCAmelCase__					:  int					   =					self.type_sequence_label_size
   UpperCAmelCase__					:  int					   =					TFDeiTForImageClassification(A    )
   UpperCAmelCase__					:  int					   =					model(A					,labels=A    )
   self.parent.assertEqual(result.logits.shape					,(self.batch_size, self.type_sequence_label_size)    )

   # test greyscale images
   UpperCAmelCase__					:  Optional[int]					   =					1
   UpperCAmelCase__					:  int					   =					TFDeiTForImageClassification(A    )

   UpperCAmelCase__					:  List[str]					   =					floats_tensor([self.batch_size, 1, self.image_size, self.image_size]    )
   UpperCAmelCase__					:  str					   =					model(A					,labels=A    )
   self.parent.assertEqual(result.logits.shape					,(self.batch_size, self.type_sequence_label_size)    )




  def    __lowercase       (  self		:			int    ):






   '''simple docstring'''
   UpperCAmelCase__					:  List[Any]					   =					self.prepare_config_and_inputs()
   UpperCAmelCase__   ,      UpperCAmelCase__   ,      UpperCAmelCase__					:  Optional[Any]					   =					config_and_inputs
   UpperCAmelCase__					:  Dict					   =					{"""pixel_values""": pixel_values}
   return config, inputs_dict







@require_tf
class      __lowercase			(    __lowerCamelCase	,			__lowerCamelCase	,			unittest.TestCase					):
  snake_case_				       =       (
      (
          TFDeiTModel,
          TFDeiTForImageClassification,
          TFDeiTForImageClassificationWithTeacher,
          TFDeiTForMaskedImageModeling,
      )
      if is_tf_available()
      else ()
  )
  snake_case_				       =       (
      {
          """feature-extraction""": TFDeiTModel,
          """image-classification""": (TFDeiTForImageClassification, TFDeiTForImageClassificationWithTeacher),
      }
      if is_tf_available()
      else {}
  )

  snake_case_				       =       False
  snake_case_				       =       False
  snake_case_				       =       False
  snake_case_				       =       False

  def    __lowercase       (  self		:			Optional[int]    ):






   '''simple docstring'''
   UpperCAmelCase__					:  List[Any]					   =					TFDeiTModelTester(self    )
   UpperCAmelCase__					:  int					   =					ConfigTester(self					,config_class=A					,has_text_modality=A					,hidden_size=37    )

  def    __lowercase       (  self		:			Union[str, Any]    ):






   '''simple docstring'''
   self.config_tester.run_common_tests()

  @unittest.skip(reason="""DeiT does not use inputs_embeds"""    )
  def    __lowercase       (  self		:			str    ):






   '''simple docstring'''
   pass

  def    __lowercase       (  self		:			Any    ):






   '''simple docstring'''
   UpperCAmelCase__   ,      UpperCAmelCase__					:  Dict					   =					self.model_tester.prepare_config_and_inputs_for_common()

   for model_class in self.all_model_classes:
    UpperCAmelCase__					:  int					   =					model_class(A    )
    self.assertIsInstance(model.get_input_embeddings()					,(tf.keras.layers.Layer)    )
    UpperCAmelCase__					:  Tuple					   =					model.get_output_embeddings()
    self.assertTrue(x is None or isinstance(A					,tf.keras.layers.Dense    )    )

  def    __lowercase       (  self		:			Tuple    ):






   '''simple docstring'''
   UpperCAmelCase__   ,      UpperCAmelCase__					:  List[Any]					   =					self.model_tester.prepare_config_and_inputs_for_common()

   for model_class in self.all_model_classes:
    UpperCAmelCase__					:  Dict					   =					model_class(A    )
    UpperCAmelCase__					:  Any					   =					inspect.signature(model.call    )
    # signature.parameters is an OrderedDict => so arg_names order is deterministic
    UpperCAmelCase__					:  List[Any]					   =					[*signature.parameters.keys()]

    UpperCAmelCase__					:  Optional[int]					   =					["""pixel_values"""]
    self.assertListEqual(arg_names[:1]					,A    )

  def    __lowercase       (  self		:			Any    ):






   '''simple docstring'''
   UpperCAmelCase__					:  List[Any]					   =					self.model_tester.prepare_config_and_inputs()
   self.model_tester.create_and_check_model(*A    )

  def    __lowercase       (  self		:			int    ):






   '''simple docstring'''
   UpperCAmelCase__					:  Union[str, Any]					   =					self.model_tester.prepare_config_and_inputs()
   self.model_tester.create_and_check_for_masked_image_modeling(*A    )

  def    __lowercase       (  self		:			List[str]    ):






   '''simple docstring'''
   UpperCAmelCase__					:  int					   =					self.model_tester.prepare_config_and_inputs()
   self.model_tester.create_and_check_for_image_classification(*A    )

  def    __lowercase       (  self		:			str					,A		:			List[str]					,A		:			Optional[int]					,A		:			Union[str, Any]=False    ):






   '''simple docstring'''
   UpperCAmelCase__					:  Any					   =					super()._prepare_for_class(A					,A					,return_labels=A    )

   if return_labels:
    if "labels" in inputs_dict and "labels" not in inspect.signature(model_class.call    ).parameters:
     del inputs_dict["labels"]

   return inputs_dict




  @slow
  def    __lowercase       (  self		:			List[str]    ):






   '''simple docstring'''
   for model_name in TF_DEIT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
    UpperCAmelCase__					:  List[Any]					   =					TFDeiTModel.from_pretrained(A    )
    self.assertIsNotNone(A    )







def   lowerCAmelCase						(					):
 '''simple docstring'''





 UpperCAmelCase__					:  Tuple					   =					Image.open("""./tests/fixtures/tests_samples/COCO/000000039769.png"""      )
 return image







@require_tf
@require_vision
class      __lowercase			(    unittest.TestCase					):

  @cached_property
  def    __lowercase       (  self		:			int    ):






   '''simple docstring'''
   return (
       DeiTImageProcessor.from_pretrained("""facebook/deit-base-distilled-patch16-224"""    )
       if is_vision_available()
       else None
   )




  @slow
  def    __lowercase       (  self		:			Optional[int]    ):






   '''simple docstring'''
   UpperCAmelCase__					:  str					   =					TFDeiTForImageClassificationWithTeacher.from_pretrained("""facebook/deit-base-distilled-patch16-224"""    )

   UpperCAmelCase__					:  Union[str, Any]					   =					self.default_image_processor
   UpperCAmelCase__					:  Optional[int]					   =					prepare_img()
   UpperCAmelCase__					:  Optional[int]					   =					image_processor(images=A					,return_tensors="""tf"""    )

   # forward pass
   UpperCAmelCase__					:  Optional[Any]					   =					model(**A    )

   # verify the logits
   UpperCAmelCase__					:  int					   =					tf.TensorShape((1, 1_000)    )
   self.assertEqual(outputs.logits.shape					,A    )

   UpperCAmelCase__					:  Any					   =					tf.constant([-1.0_2_6_6, 0.1_9_1_2, -1.2_8_6_1]    )

   self.assertTrue(np.allclose(outputs.logits[0, :3]					,A					,atol=1e-4    )    )





