


'''simple docstring'''






from __future__ import annotations

import copy
import inspect
import json
import math
import os
import tempfile
import unittest
from importlib import import_module

import numpy as np

from transformers import ViTMAEConfig
from transformers.file_utils import cached_property, is_tf_available, is_vision_available
from transformers.testing_utils import require_tf, require_vision, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
		import tensorflow as tf

		from transformers import TFViTMAEForPreTraining, TFViTMAEModel


if is_vision_available():
		from PIL import Image

		from transformers import ViTImageProcessor




class 	a__       :





							'''simple docstring'''






							def __init__(	self      ,					lowerCamelCase_      ,					lowerCamelCase_=13      ,					lowerCamelCase_=30      ,					lowerCamelCase_=2      ,					lowerCamelCase_=3      ,					lowerCamelCase_=True      ,					lowerCamelCase_=True      ,					lowerCamelCase_=32      ,					lowerCamelCase_=2      ,					lowerCamelCase_=4      ,					lowerCamelCase_=37      ,					lowerCamelCase_="gelu"      ,					lowerCamelCase_=0.1      ,					lowerCamelCase_=0.1      ,					lowerCamelCase_=10      ,					lowerCamelCase_=0.02      ,					lowerCamelCase_=3      ,					lowerCamelCase_=0.6      ,					lowerCamelCase_=None      ,					) ->							Dict:
														lowerCAmelCase__      =			parent
														lowerCAmelCase__      =			batch_size
														lowerCAmelCase__      =			image_size
														lowerCAmelCase__      =			patch_size
														lowerCAmelCase__      =			num_channels
														lowerCAmelCase__      =			is_training
														lowerCAmelCase__      =			use_labels
														lowerCAmelCase__      =			hidden_size
														lowerCAmelCase__      =			num_hidden_layers
														lowerCAmelCase__      =			num_attention_heads
														lowerCAmelCase__      =			intermediate_size
														lowerCAmelCase__      =			hidden_act
														lowerCAmelCase__      =			hidden_dropout_prob
														lowerCAmelCase__      =			attention_probs_dropout_prob
														lowerCAmelCase__      =			type_sequence_label_size
														lowerCAmelCase__      =			initializer_range
														lowerCAmelCase__      =			mask_ratio
														lowerCAmelCase__      =			scope

														# in ViTMAE, the expected sequence length = (num_patches + 1) * (1 - config.mask_ratio), rounded above
														# (we add 1 for the [CLS] token)
														lowerCAmelCase__      =			(image_size // patch_size) ** 2
														lowerCAmelCase__      =			int(math.ceil((1 - mask_ratio) * (num_patches + 1)     )     )







							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							Dict:
														lowerCAmelCase__      =			floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]     )

														lowerCAmelCase__      =			None
														if self.use_labels:
																					lowerCAmelCase__      =			ids_tensor([self.batch_size]      ,					self.type_sequence_label_size     )

														lowerCAmelCase__      =			self.get_config()

														return config, pixel_values, labels







							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							Any:
														return ViTMAEConfig(
														    image_size=self.image_size      ,					patch_size=self.patch_size      ,					num_channels=self.num_channels      ,					hidden_size=self.hidden_size      ,					num_hidden_layers=self.num_hidden_layers      ,					num_attention_heads=self.num_attention_heads      ,					intermediate_size=self.intermediate_size      ,					decoder_hidden_size=self.hidden_size      ,					decoder_num_hidden_layers=self.num_hidden_layers      ,					decoder_num_attention_heads=self.num_attention_heads      ,					decoder_intermediate_size=self.intermediate_size      ,					hidden_act=self.hidden_act      ,					hidden_dropout_prob=self.hidden_dropout_prob      ,					attention_probs_dropout_prob=self.attention_probs_dropout_prob      ,					is_decoder=lowerCamelCase_      ,					initializer_range=self.initializer_range      ,					mask_ratio=self.mask_ratio      ,					)







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_      ,					lowerCamelCase_      ,					lowerCamelCase_     ) ->							List[str]:
														lowerCAmelCase__      =			TFViTMAEModel(config=lowerCamelCase_     )
														lowerCAmelCase__      =			model(lowerCamelCase_      ,					training=lowerCamelCase_     )
														self.parent.assertEqual(result.last_hidden_state.shape      ,					(self.batch_size, self.seq_length, self.hidden_size)     )







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_      ,					lowerCamelCase_      ,					lowerCamelCase_     ) ->							int:
														lowerCAmelCase__      =			TFViTMAEForPreTraining(lowerCamelCase_     )
														lowerCAmelCase__      =			model(lowerCamelCase_      ,					training=lowerCamelCase_     )
														# expected sequence length = num_patches
														lowerCAmelCase__      =			(self.image_size // self.patch_size) ** 2
														lowerCAmelCase__      =			self.patch_size**2 * self.num_channels
														self.parent.assertEqual(result.logits.shape      ,					(self.batch_size, num_patches, expected_num_channels)     )

														# test greyscale images
														lowerCAmelCase__      =			1
														lowerCAmelCase__      =			TFViTMAEForPreTraining(lowerCamelCase_     )

														lowerCAmelCase__      =			floats_tensor([self.batch_size, 1, self.image_size, self.image_size]     )
														lowerCAmelCase__      =			model(lowerCamelCase_      ,					training=lowerCamelCase_     )
														lowerCAmelCase__      =			self.patch_size**2
														self.parent.assertEqual(result.logits.shape      ,					(self.batch_size, num_patches, expected_num_channels)     )







							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							List[Any]:
														lowerCAmelCase__      =			self.prepare_config_and_inputs()
														((lowerCAmelCase__)     ,      (lowerCAmelCase__)     ,      (lowerCAmelCase__))      =			config_and_inputs
														lowerCAmelCase__      =			{'''pixel_values''': pixel_values}
														return config, inputs_dict




@require_tf
class 	a__       (							a__       ,       a__       ,       unittest.TestCase						):





							'''simple docstring'''






							lowercase__    :     str					=   (TFViTMAEModel, TFViTMAEForPreTraining) if is_tf_available() else ()
							lowercase__    :     Any					=   {"feature-extraction": TFViTMAEModel} if is_tf_available() else {}

							lowercase__    :     str					=   False
							lowercase__    :     Union[str, Any]					=   False
							lowercase__    :     int					=   False
							lowercase__    :     Union[str, Any]					=   False







							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							int:
														lowerCAmelCase__      =			TFViTMAEModelTester(self     )
														lowerCAmelCase__      =			ConfigTester(self      ,					config_class=lowerCamelCase_      ,					has_text_modality=lowerCamelCase_      ,					hidden_size=37     )







							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							Optional[Any]:
														self.config_tester.run_common_tests()







							@unittest.skip(reason='''ViTMAE does not use inputs_embeds'''     )
							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							List[str]:
														pass







							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							str:
														lowerCAmelCase__     ,      lowerCAmelCase__      =			self.model_tester.prepare_config_and_inputs_for_common()

														for model_class in self.all_model_classes:
																					lowerCAmelCase__      =			model_class(lowerCamelCase_     )
																					self.assertIsInstance(model.get_input_embeddings()      ,					(tf.keras.layers.Layer)     )
																					lowerCAmelCase__      =			model.get_output_embeddings()
																					self.assertTrue(x is None or isinstance(lowerCamelCase_      ,					tf.keras.layers.Layer     )     )







							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							str:
														lowerCAmelCase__     ,      lowerCAmelCase__      =			self.model_tester.prepare_config_and_inputs_for_common()

														for model_class in self.all_model_classes:
																					lowerCAmelCase__      =			model_class(lowerCamelCase_     )
																					lowerCAmelCase__      =			inspect.signature(model.call     )
																					# signature.parameters is an OrderedDict => so arg_names order is deterministic
																					lowerCAmelCase__      =			[*signature.parameters.keys()]

																					lowerCAmelCase__      =			['''pixel_values''']
																					self.assertListEqual(arg_names[:1]      ,					lowerCamelCase_     )







							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							Union[str, Any]:
														lowerCAmelCase__      =			self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_model(*lowerCamelCase_     )







							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							Optional[Any]:
														lowerCAmelCase__      =			self.model_tester.prepare_config_and_inputs()
														self.model_tester.create_and_check_for_pretraining(*lowerCamelCase_     )







							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							int:
														# make the mask reproducible
														np.random.seed(2     )

														lowerCAmelCase__     ,      lowerCAmelCase__      =			self.model_tester.prepare_config_and_inputs_for_common()

														lowerCAmelCase__      =			int((config.image_size // config.patch_size) ** 2     )
														lowerCAmelCase__      =			np.random.uniform(size=(self.model_tester.batch_size, num_patches)     )

														for model_class in self.all_model_classes:
																					lowerCAmelCase__      =			model_class(lowerCamelCase_     )
																					lowerCAmelCase__      =			self._prepare_for_class(lowerCamelCase_      ,					lowerCamelCase_     )

																					lowerCAmelCase__      =			model(lowerCamelCase_      ,					noise=lowerCamelCase_     )

																					lowerCAmelCase__      =			copy.deepcopy(self._prepare_for_class(lowerCamelCase_      ,					lowerCamelCase_     )     )
																					lowerCAmelCase__      =			model(**lowerCamelCase_      ,					noise=lowerCamelCase_     )
																					lowerCAmelCase__      =			outputs_dict[0].numpy()
																					lowerCAmelCase__      =			outputs_keywords[0].numpy()

																					self.assertLess(np.sum(np.abs(output_dict - output_keywords     )     )      ,					1e-6     )







							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							Optional[int]:
														# make the mask reproducible
														np.random.seed(2     )

														lowerCAmelCase__     ,      lowerCAmelCase__      =			self.model_tester.prepare_config_and_inputs_for_common()

														lowerCAmelCase__      =			int((config.image_size // config.patch_size) ** 2     )
														lowerCAmelCase__      =			np.random.uniform(size=(self.model_tester.batch_size, num_patches)     )

														def prepare_numpy_arrays(lowerCamelCase_     ):
																					lowerCAmelCase__      =			{}
																					for k, v in inputs_dict.items():
																												if tf.is_tensor(lowerCamelCase_     ):
																																			lowerCAmelCase__      =			v.numpy()
																												else:
																																			lowerCAmelCase__      =			np.array(lowerCamelCase_     )

																					return inputs_np_dict

														for model_class in self.all_model_classes:
																					lowerCAmelCase__      =			model_class(lowerCamelCase_     )

																					lowerCAmelCase__      =			self._prepare_for_class(lowerCamelCase_      ,					lowerCamelCase_     )
																					lowerCAmelCase__      =			prepare_numpy_arrays(lowerCamelCase_     )

																					lowerCAmelCase__      =			model(lowerCamelCase_      ,					noise=lowerCamelCase_     )
																					lowerCAmelCase__      =			model(**lowerCamelCase_      ,					noise=lowerCamelCase_     )
																					self.assert_outputs_same(lowerCamelCase_      ,					lowerCamelCase_     )







							def 						__SCREAMING_SNAKE_CASE							(	self      ,					lowerCamelCase_      ,					lowerCamelCase_      ,					lowerCamelCase_     ) ->							Union[str, Any]:
														# make masks reproducible
														np.random.seed(2     )

														lowerCAmelCase__      =			int((tf_model.config.image_size // tf_model.config.patch_size) ** 2     )
														lowerCAmelCase__      =			np.random.uniform(size=(self.model_tester.batch_size, num_patches)     )
														lowerCAmelCase__      =			tf.constant(lowerCamelCase_     )

														# Add `noise` argument.
														# PT inputs will be prepared in `super().check_pt_tf_models()` with this added `noise` argument
														lowerCAmelCase__      =			tf_noise

														super().check_pt_tf_models(lowerCamelCase_      ,					lowerCamelCase_      ,					lowerCamelCase_     )







							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							Dict:
														# make mask reproducible
														np.random.seed(2     )

														lowerCAmelCase__     ,      lowerCAmelCase__      =			self.model_tester.prepare_config_and_inputs_for_common()

														lowerCAmelCase__      =			{
														    module_member
														    for model_class in self.all_model_classes
														    for module in (import_module(model_class.__module__     ),)
														    for module_member_name in dir(lowerCamelCase_     )
														    if module_member_name.endswith('''MainLayer'''     )
														    # This condition is required, since `modeling_tf_clip.py` has 3 classes whose names end with `MainLayer`.
														    and module_member_name[: -len('''MainLayer'''     )] == model_class.__name__[: -len('''Model'''     )]
														    for module_member in (getattr(lowerCamelCase_      ,					lowerCamelCase_     ),)
														    if isinstance(lowerCamelCase_      ,					lowerCamelCase_     )
														    and tf.keras.layers.Layer in module_member.__bases__
														    and getattr(lowerCamelCase_      ,					'''_keras_serializable'''      ,					lowerCamelCase_     )
														}

														lowerCAmelCase__      =			int((config.image_size // config.patch_size) ** 2     )
														lowerCAmelCase__      =			np.random.uniform(size=(self.model_tester.batch_size, num_patches)     )
														lowerCAmelCase__      =			tf.convert_to_tensor(lowerCamelCase_     )
														inputs_dict.update({'''noise''': noise}     )

														for main_layer_class in tf_main_layer_classes:
																					lowerCAmelCase__      =			main_layer_class(lowerCamelCase_     )

																					lowerCAmelCase__      =			{
																					    name: tf.keras.Input(tensor.shape[1:]      ,					dtype=tensor.dtype     ) for name, tensor in inputs_dict.items()
																					}

																					lowerCAmelCase__      =			tf.keras.Model(lowerCamelCase_      ,					outputs=main_layer(lowerCamelCase_     )     )
																					lowerCAmelCase__      =			model(lowerCamelCase_     )

																					with tempfile.TemporaryDirectory() as tmpdirname:
																												lowerCAmelCase__      =			os.path.join(lowerCamelCase_      ,					'''keras_model.h5'''     )
																												model.save(lowerCamelCase_     )
																												lowerCAmelCase__      =			tf.keras.models.load_model(
																												    lowerCamelCase_      ,					custom_objects={main_layer_class.__name__: main_layer_class}     )
																												assert isinstance(lowerCamelCase_      ,					tf.keras.Model     )
																												lowerCAmelCase__      =			model(lowerCamelCase_     )
																												self.assert_outputs_same(lowerCamelCase_      ,					lowerCamelCase_     )







							@slow
							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							Tuple:
														# make mask reproducible
														np.random.seed(2     )

														lowerCAmelCase__     ,      lowerCAmelCase__      =			self.model_tester.prepare_config_and_inputs_for_common()

														lowerCAmelCase__      =			int((config.image_size // config.patch_size) ** 2     )
														lowerCAmelCase__      =			np.random.uniform(size=(self.model_tester.batch_size, num_patches)     )

														for model_class in self.all_model_classes:
																					lowerCAmelCase__      =			model_class(lowerCamelCase_     )
																					lowerCAmelCase__      =			self._prepare_for_class(lowerCamelCase_      ,					lowerCamelCase_     )
																					lowerCAmelCase__      =			model(lowerCamelCase_      ,					noise=lowerCamelCase_     )

																					if model_class.__name__ == "TFViTMAEModel":
																												lowerCAmelCase__      =			outputs.last_hidden_state.numpy()
																												lowerCAmelCase__      =			0
																					else:
																												lowerCAmelCase__      =			outputs.logits.numpy()
																												lowerCAmelCase__      =			0

																					with tempfile.TemporaryDirectory() as tmpdirname:
																												model.save_pretrained(lowerCamelCase_      ,					saved_model=lowerCamelCase_     )
																												lowerCAmelCase__      =			model_class.from_pretrained(lowerCamelCase_     )
																												lowerCAmelCase__      =			model(lowerCamelCase_      ,					noise=lowerCamelCase_     )

																												if model_class.__name__ == "TFViTMAEModel":
																																			lowerCAmelCase__      =			after_outputs['''last_hidden_state'''].numpy()
																																			lowerCAmelCase__      =			0
																												else:
																																			lowerCAmelCase__      =			after_outputs['''logits'''].numpy()
																																			lowerCAmelCase__      =			0

																												lowerCAmelCase__      =			np.amax(np.abs(out_a - out_a     )     )
																												self.assertLessEqual(lowerCamelCase_      ,					1e-5     )







							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							int:
														# make mask reproducible
														np.random.seed(2     )

														lowerCAmelCase__     ,      lowerCAmelCase__      =			self.model_tester.prepare_config_and_inputs_for_common()

														lowerCAmelCase__      =			int((config.image_size // config.patch_size) ** 2     )
														lowerCAmelCase__      =			np.random.uniform(size=(self.model_tester.batch_size, num_patches)     )

														for model_class in self.all_model_classes:
																					lowerCAmelCase__      =			model_class(lowerCamelCase_     )
																					lowerCAmelCase__      =			self._prepare_for_class(lowerCamelCase_      ,					lowerCamelCase_     )

																					lowerCAmelCase__      =			model(lowerCamelCase_      ,					noise=lowerCamelCase_     )
																					lowerCAmelCase__      =			model.get_config()
																					# make sure that returned config is jsonifiable, which is required by keras
																					json.dumps(lowerCamelCase_     )
																					lowerCAmelCase__      =			model_class.from_config(model.get_config()     )
																					# make sure it also accepts a normal config
																					lowerCAmelCase__      =			model_class.from_config(model.config     )
																					lowerCAmelCase__      =			new_model(lowerCamelCase_     )  # Build model
																					new_model.set_weights(model.get_weights()     )
																					lowerCAmelCase__      =			new_model(lowerCamelCase_      ,					noise=lowerCamelCase_     )

																					self.assert_outputs_same(lowerCamelCase_      ,					lowerCamelCase_     )







							@unittest.skip(
							    reason='''ViTMAE returns a random mask + ids_restore in each forward pass. See test_save_load
    to get deterministic results.'''     )
							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							Optional[int]:
														pass







							@unittest.skip(reason='''ViTMAE returns a random mask + ids_restore in each forward pass. See test_save_load'''     )
							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							Union[str, Any]:
														pass







							@slow
							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							Dict:
														lowerCAmelCase__      =			TFViTMAEModel.from_pretrained('''google/vit-base-patch16-224'''     )
														self.assertIsNotNone(lowerCamelCase_     )


def       _snake_case     (  )		->					Optional[int]:
							lowerCAmelCase__      =			Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png''' )
							return image






@require_tf
@require_vision
class 	a__       (							unittest.TestCase						):





							'''simple docstring'''






							@cached_property
							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							str:
														return ViTImageProcessor.from_pretrained('''facebook/vit-mae-base'''     ) if is_vision_available() else None







							@slow
							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							Tuple:
														# make random mask reproducible across the PT and TF model
														np.random.seed(2     )

														lowerCAmelCase__      =			TFViTMAEForPreTraining.from_pretrained('''facebook/vit-mae-base'''     )

														lowerCAmelCase__      =			self.default_image_processor
														lowerCAmelCase__      =			prepare_img()
														lowerCAmelCase__      =			image_processor(images=lowerCamelCase_      ,					return_tensors='''tf'''     )

														# prepare a noise vector that will be also used for testing the TF model
														# (this way we can ensure that the PT and TF models operate on the same inputs)
														lowerCAmelCase__      =			ViTMAEConfig()
														lowerCAmelCase__      =			int((vit_mae_config.image_size // vit_mae_config.patch_size) ** 2     )
														lowerCAmelCase__      =			np.random.uniform(size=(1, num_patches)     )

														# forward pass
														lowerCAmelCase__      =			model(**lowerCamelCase_      ,					noise=lowerCamelCase_     )

														# verify the logits
														lowerCAmelCase__      =			tf.convert_to_tensor([1, 1_96, 7_68]     )
														self.assertEqual(outputs.logits.shape      ,					lowerCamelCase_     )

														lowerCAmelCase__      =			tf.convert_to_tensor(
														    [[-0.0_548, -1.7_023, -0.9_325], [0.3_721, -0.5_670, -0.2_233], [0.8_235, -1.3_878, -0.3_524]]     )

														tf.debugging.assert_near(outputs.logits[0, :3, :3]      ,					lowerCamelCase_      ,					atol=1e-4     )



'''simple docstring'''






import contextlib
import csv
import json
import os
import sqlitea
import tarfile
import textwrap
import zipfile

import pyarrow as pa
import pyarrow.parquet as pq
import pytest

import datasets
import datasets.config


@pytest.fixture(scope='''session''' )
def       _snake_case     (  )		->					Any:
							lowerCAmelCase__      =			10
							lowerCAmelCase__      =			datasets.Features(
							    {
							        '''tokens''': datasets.Sequence(datasets.Value('''string''' ) ),
							        '''labels''': datasets.Sequence(datasets.ClassLabel(names=['''negative''', '''positive'''] ) ),
							        '''answers''': datasets.Sequence(
							            {
							                '''text''': datasets.Value('''string''' ),
							                '''answer_start''': datasets.Value('''int32''' ),
							            } ),
							        '''id''': datasets.Value('''int64''' ),
							    } )
							lowerCAmelCase__      =			datasets.Dataset.from_dict(
							    {
							        '''tokens''': [['''foo'''] * 5] * n,
							        '''labels''': [[1] * 5] * n,
							        '''answers''': [{'''answer_start''': [97], '''text''': ['''1976''']}] * 10,
							        '''id''': list(range(A ) ),
							    }				,					features=A				,					)
							return dataset


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A				,					A )		->					List[Any]:
							lowerCAmelCase__      =			str(tmp_path_factory.mktemp('''data''' ) / '''file.arrow''' )
							dataset.map(cache_file_name=A )
							return filename


# FILE_CONTENT + files


__UpperCAmelCase					       =							'''\
    Text data.
    Second line of data.'''


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A )		->					Optional[int]:
							lowerCAmelCase__      =			tmp_path_factory.mktemp('''data''' ) / '''file.txt'''
							lowerCAmelCase__      =			FILE_CONTENT
							with open(A				,					'''w''' ) as f:
														f.write(A )
							return filename


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A )		->					Union[str, Any]:
							import bza

							lowerCAmelCase__      =			tmp_path_factory.mktemp('''data''' ) / '''file.txt.bz2'''
							lowerCAmelCase__      =			bytes(A				,					'''utf-8''' )
							with bza.open(A				,					'''wb''' ) as f:
														f.write(A )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A )		->					Tuple:
							import gzip

							lowerCAmelCase__      =			str(tmp_path_factory.mktemp('''data''' ) / '''file.txt.gz''' )
							lowerCAmelCase__      =			bytes(A				,					'''utf-8''' )
							with gzip.open(A				,					'''wb''' ) as f:
														f.write(A )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A )		->					List[str]:
							if datasets.config.LZ4_AVAILABLE:
														import lza.frame

														lowerCAmelCase__      =			tmp_path_factory.mktemp('''data''' ) / '''file.txt.lz4'''
														lowerCAmelCase__      =			bytes(A				,					'''utf-8''' )
														with lza.frame.open(A				,					'''wb''' ) as f:
																					f.write(A )
														return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A				,					A )		->					int:
							if datasets.config.PY7ZR_AVAILABLE:
														import pyazr

														lowerCAmelCase__      =			tmp_path_factory.mktemp('''data''' ) / '''file.txt.7z'''
														with pyazr.SevenZipFile(A				,					'''w''' ) as archive:
																					archive.write(A				,					arcname=os.path.basename(A ) )
														return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A				,					A )		->					str:
							import tarfile

							lowerCAmelCase__      =			tmp_path_factory.mktemp('''data''' ) / '''file.txt.tar'''
							with tarfile.TarFile(A				,					'''w''' ) as f:
														f.add(A				,					arcname=os.path.basename(A ) )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A )		->					Optional[int]:
							import lzma

							lowerCAmelCase__      =			tmp_path_factory.mktemp('''data''' ) / '''file.txt.xz'''
							lowerCAmelCase__      =			bytes(A				,					'''utf-8''' )
							with lzma.open(A				,					'''wb''' ) as f:
														f.write(A )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A				,					A )		->					List[str]:
							import zipfile

							lowerCAmelCase__      =			tmp_path_factory.mktemp('''data''' ) / '''file.txt.zip'''
							with zipfile.ZipFile(A				,					'''w''' ) as f:
														f.write(A				,					arcname=os.path.basename(A ) )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A )		->					Any:
							if datasets.config.ZSTANDARD_AVAILABLE:
														import zstandard as zstd

														lowerCAmelCase__      =			tmp_path_factory.mktemp('''data''' ) / '''file.txt.zst'''
														lowerCAmelCase__      =			bytes(A				,					'''utf-8''' )
														with zstd.open(A				,					'''wb''' ) as f:
																					f.write(A )
														return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A )		->					Any:
							lowerCAmelCase__      =			tmp_path_factory.mktemp('''data''' ) / '''file.xml'''
							lowerCAmelCase__      =			textwrap.dedent(
							    '''\
    <?xml version="1.0" encoding="UTF-8" ?>
    <tmx version="1.4">
      <header segtype="sentence" srclang="ca" />
      <body>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 1</seg></tuv>
          <tuv xml:lang="en"><seg>Content 1</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 2</seg></tuv>
          <tuv xml:lang="en"><seg>Content 2</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 3</seg></tuv>
          <tuv xml:lang="en"><seg>Content 3</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 4</seg></tuv>
          <tuv xml:lang="en"><seg>Content 4</seg></tuv>
        </tu>
        <tu>
          <tuv xml:lang="ca"><seg>Contingut 5</seg></tuv>
          <tuv xml:lang="en"><seg>Content 5</seg></tuv>
        </tu>
      </body>
    </tmx>''' )
							with open(A				,					'''w''' ) as f:
														f.write(A )
							return filename


__UpperCAmelCase					       =							[
    {'''col_1''': '''0''', '''col_2''': 0, '''col_3''': 0.0},
    {'''col_1''': '''1''', '''col_2''': 1, '''col_3''': 1.0},
    {'''col_1''': '''2''', '''col_2''': 2, '''col_3''': 2.0},
    {'''col_1''': '''3''', '''col_2''': 3, '''col_3''': 3.0},
]
__UpperCAmelCase					       =							[
    {'''col_1''': '''4''', '''col_2''': 4, '''col_3''': 4.0},
    {'''col_1''': '''5''', '''col_2''': 5, '''col_3''': 5.0},
]
__UpperCAmelCase					       =							{
    '''col_1''': ['''0''', '''1''', '''2''', '''3'''],
    '''col_2''': [0, 1, 2, 3],
    '''col_3''': [0.0, 1.0, 2.0, 3.0],
}

__UpperCAmelCase					       =							[
    {'''col_3''': 0.0, '''col_1''': '''0''', '''col_2''': 0},
    {'''col_3''': 1.0, '''col_1''': '''1''', '''col_2''': 1},
]

__UpperCAmelCase					       =							[
    {'''col_1''': '''s0''', '''col_2''': 0, '''col_3''': 0.0},
    {'''col_1''': '''s1''', '''col_2''': 1, '''col_3''': 1.0},
    {'''col_1''': '''s2''', '''col_2''': 2, '''col_3''': 2.0},
    {'''col_1''': '''s3''', '''col_2''': 3, '''col_3''': 3.0},
]


@pytest.fixture(scope='''session''' )
def       _snake_case     (  )		->					Union[str, Any]:
							return DATA_DICT_OF_LISTS


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A )		->					Optional[Any]:
							lowerCAmelCase__      =			datasets.Dataset.from_dict(A )
							lowerCAmelCase__      =			str(tmp_path_factory.mktemp('''data''' ) / '''dataset.arrow''' )
							dataset.map(cache_file_name=A )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A )		->					str:
							lowerCAmelCase__      =			str(tmp_path_factory.mktemp('''data''' ) / '''dataset.sqlite''' )
							with contextlib.closing(sqlitea.connect(A ) ) as con:
														lowerCAmelCase__      =			con.cursor()
														cur.execute('''CREATE TABLE dataset(col_1 text, col_2 int, col_3 real)''' )
														for item in DATA:
																					cur.execute('''INSERT INTO dataset(col_1, col_2, col_3) VALUES (?, ?, ?)'''				,					tuple(item.values() ) )
														con.commit()
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A )		->					Any:
							lowerCAmelCase__      =			str(tmp_path_factory.mktemp('''data''' ) / '''dataset.csv''' )
							with open(A				,					'''w'''				,					newline='''''' ) as f:
														lowerCAmelCase__      =			csv.DictWriter(A				,					fieldnames=['''col_1''', '''col_2''', '''col_3'''] )
														writer.writeheader()
														for item in DATA:
																					writer.writerow(A )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A )		->					Optional[int]:
							lowerCAmelCase__      =			str(tmp_path_factory.mktemp('''data''' ) / '''dataset2.csv''' )
							with open(A				,					'''w'''				,					newline='''''' ) as f:
														lowerCAmelCase__      =			csv.DictWriter(A				,					fieldnames=['''col_1''', '''col_2''', '''col_3'''] )
														writer.writeheader()
														for item in DATA:
																					writer.writerow(A )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A				,					A )		->					Optional[int]:
							import bza

							lowerCAmelCase__      =			tmp_path_factory.mktemp('''data''' ) / '''dataset.csv.bz2'''
							with open(A				,					'''rb''' ) as f:
														lowerCAmelCase__      =			f.read()
							# data = bytes(FILE_CONTENT, "utf-8")
							with bza.open(A				,					'''wb''' ) as f:
														f.write(A )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A				,					A				,					A )		->					List[str]:
							lowerCAmelCase__      =			tmp_path_factory.mktemp('''data''' ) / '''dataset.csv.zip'''
							with zipfile.ZipFile(A				,					'''w''' ) as f:
														f.write(A				,					arcname=os.path.basename(A ) )
														f.write(A				,					arcname=os.path.basename(A ) )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A				,					A				,					A )		->					Dict:
							lowerCAmelCase__      =			tmp_path_factory.mktemp('''data''' ) / '''dataset.csv.zip'''
							with zipfile.ZipFile(A				,					'''w''' ) as f:
														f.write(A				,					arcname=os.path.basename(csv_path.replace('''.csv'''				,					'''.CSV''' ) ) )
														f.write(A				,					arcname=os.path.basename(csva_path.replace('''.csv'''				,					'''.CSV''' ) ) )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A				,					A				,					A )		->					Optional[int]:
							lowerCAmelCase__      =			tmp_path_factory.mktemp('''data''' ) / '''dataset_with_dir.csv.zip'''
							with zipfile.ZipFile(A				,					'''w''' ) as f:
														f.write(A				,					arcname=os.path.join('''main_dir'''				,					os.path.basename(A ) ) )
														f.write(A				,					arcname=os.path.join('''main_dir'''				,					os.path.basename(A ) ) )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A )		->					str:
							lowerCAmelCase__      =			str(tmp_path_factory.mktemp('''data''' ) / '''dataset.parquet''' )
							lowerCAmelCase__      =			pa.schema(
							    {
							        '''col_1''': pa.string(),
							        '''col_2''': pa.intaa(),
							        '''col_3''': pa.floataa(),
							    } )
							with open(A				,					'''wb''' ) as f:
														lowerCAmelCase__      =			pq.ParquetWriter(A				,					schema=A )
														lowerCAmelCase__      =			pa.Table.from_pydict({k: [DATA[i][k] for i in range(len(A ) )] for k in DATA[0]}				,					schema=A )
														writer.write_table(A )
														writer.close()
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A )		->					str:
							lowerCAmelCase__      =			str(tmp_path_factory.mktemp('''data''' ) / '''dataset.json''' )
							lowerCAmelCase__      =			{'''data''': DATA}
							with open(A				,					'''w''' ) as f:
														json.dump(A				,					A )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A )		->					Union[str, Any]:
							lowerCAmelCase__      =			str(tmp_path_factory.mktemp('''data''' ) / '''dataset.json''' )
							lowerCAmelCase__      =			{'''data''': DATA_DICT_OF_LISTS}
							with open(A				,					'''w''' ) as f:
														json.dump(A				,					A )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A )		->					str:
							lowerCAmelCase__      =			str(tmp_path_factory.mktemp('''data''' ) / '''dataset.jsonl''' )
							with open(A				,					'''w''' ) as f:
														for item in DATA:
																					f.write(json.dumps(A ) + '''\n''' )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A )		->					Optional[Any]:
							lowerCAmelCase__      =			str(tmp_path_factory.mktemp('''data''' ) / '''dataset2.jsonl''' )
							with open(A				,					'''w''' ) as f:
														for item in DATA:
																					f.write(json.dumps(A ) + '''\n''' )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A )		->					List[str]:
							lowerCAmelCase__      =			str(tmp_path_factory.mktemp('''data''' ) / '''dataset_312.jsonl''' )
							with open(A				,					'''w''' ) as f:
														for item in DATA_312:
																					f.write(json.dumps(A ) + '''\n''' )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A )		->					List[Any]:
							lowerCAmelCase__      =			str(tmp_path_factory.mktemp('''data''' ) / '''dataset-str.jsonl''' )
							with open(A				,					'''w''' ) as f:
														for item in DATA_STR:
																					f.write(json.dumps(A ) + '''\n''' )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A				,					A )		->					List[str]:
							import gzip

							lowerCAmelCase__      =			str(tmp_path_factory.mktemp('''data''' ) / '''dataset.txt.gz''' )
							with open(A				,					'''rb''' ) as orig_file:
														with gzip.open(A				,					'''wb''' ) as zipped_file:
																					zipped_file.writelines(A )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A				,					A )		->					Dict:
							import gzip

							lowerCAmelCase__      =			str(tmp_path_factory.mktemp('''data''' ) / '''dataset.jsonl.gz''' )
							with open(A				,					'''rb''' ) as orig_file:
														with gzip.open(A				,					'''wb''' ) as zipped_file:
																					zipped_file.writelines(A )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A				,					A				,					A )		->					str:
							lowerCAmelCase__      =			tmp_path_factory.mktemp('''data''' ) / '''dataset.jsonl.zip'''
							with zipfile.ZipFile(A				,					'''w''' ) as f:
														f.write(A				,					arcname=os.path.basename(A ) )
														f.write(A				,					arcname=os.path.basename(A ) )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A				,					A				,					A				,					A )		->					Optional[Any]:
							lowerCAmelCase__      =			tmp_path_factory.mktemp('''data''' ) / '''dataset_nested.jsonl.zip'''
							with zipfile.ZipFile(A				,					'''w''' ) as f:
														f.write(A				,					arcname=os.path.join('''nested'''				,					os.path.basename(A ) ) )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A				,					A				,					A )		->					List[Any]:
							lowerCAmelCase__      =			tmp_path_factory.mktemp('''data''' ) / '''dataset_with_dir.jsonl.zip'''
							with zipfile.ZipFile(A				,					'''w''' ) as f:
														f.write(A				,					arcname=os.path.join('''main_dir'''				,					os.path.basename(A ) ) )
														f.write(A				,					arcname=os.path.join('''main_dir'''				,					os.path.basename(A ) ) )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A				,					A				,					A )		->					List[str]:
							lowerCAmelCase__      =			tmp_path_factory.mktemp('''data''' ) / '''dataset.jsonl.tar'''
							with tarfile.TarFile(A				,					'''w''' ) as f:
														f.add(A				,					arcname=os.path.basename(A ) )
														f.add(A				,					arcname=os.path.basename(A ) )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A				,					A				,					A				,					A )		->					Optional[int]:
							lowerCAmelCase__      =			tmp_path_factory.mktemp('''data''' ) / '''dataset_nested.jsonl.tar'''
							with tarfile.TarFile(A				,					'''w''' ) as f:
														f.add(A				,					arcname=os.path.join('''nested'''				,					os.path.basename(A ) ) )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A )		->					Any:
							lowerCAmelCase__      =			['''0''', '''1''', '''2''', '''3''']
							lowerCAmelCase__      =			str(tmp_path_factory.mktemp('''data''' ) / '''dataset.txt''' )
							with open(A				,					'''w''' ) as f:
														for item in data:
																					f.write(item + '''\n''' )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A )		->					Union[str, Any]:
							lowerCAmelCase__      =			['''0''', '''1''', '''2''', '''3''']
							lowerCAmelCase__      =			str(tmp_path_factory.mktemp('''data''' ) / '''dataset2.txt''' )
							with open(A				,					'''w''' ) as f:
														for item in data:
																					f.write(item + '''\n''' )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A )		->					Union[str, Any]:
							lowerCAmelCase__      =			['''0''', '''1''', '''2''', '''3''']
							lowerCAmelCase__      =			tmp_path_factory.mktemp('''data''' ) / '''dataset.abc'''
							with open(A				,					'''w''' ) as f:
														for item in data:
																					f.write(item + '''\n''' )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A				,					A				,					A )		->					List[Any]:
							lowerCAmelCase__      =			tmp_path_factory.mktemp('''data''' ) / '''dataset.text.zip'''
							with zipfile.ZipFile(A				,					'''w''' ) as f:
														f.write(A				,					arcname=os.path.basename(A ) )
														f.write(A				,					arcname=os.path.basename(A ) )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A				,					A				,					A )		->					Union[str, Any]:
							lowerCAmelCase__      =			tmp_path_factory.mktemp('''data''' ) / '''dataset_with_dir.text.zip'''
							with zipfile.ZipFile(A				,					'''w''' ) as f:
														f.write(A				,					arcname=os.path.join('''main_dir'''				,					os.path.basename(A ) ) )
														f.write(A				,					arcname=os.path.join('''main_dir'''				,					os.path.basename(A ) ) )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A				,					A				,					A )		->					Optional[int]:
							lowerCAmelCase__      =			tmp_path_factory.mktemp('''data''' ) / '''dataset.ext.zip'''
							with zipfile.ZipFile(A				,					'''w''' ) as f:
														f.write(A				,					arcname=os.path.basename('''unsupported.ext''' ) )
														f.write(A				,					arcname=os.path.basename('''unsupported_2.ext''' ) )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A )		->					Tuple:
							lowerCAmelCase__      =			'''\n'''.join(['''First''', '''Second\u2029with Unicode new line''', '''Third'''] )
							lowerCAmelCase__      =			str(tmp_path_factory.mktemp('''data''' ) / '''dataset_with_unicode_new_lines.txt''' )
							with open(A				,					'''w'''				,					encoding='''utf-8''' ) as f:
														f.write(A )
							return path


@pytest.fixture(scope='''session''' )
def       _snake_case     (  )		->					List[str]:
							return os.path.join('''tests'''				,					'''features'''				,					'''data'''				,					'''test_image_rgb.jpg''' )


@pytest.fixture(scope='''session''' )
def       _snake_case     (  )		->					Any:
							return os.path.join('''tests'''				,					'''features'''				,					'''data'''				,					'''test_audio_44100.wav''' )


@pytest.fixture(scope='''session''' )
def       _snake_case     (  A				,					A )		->					Union[str, Any]:
							lowerCAmelCase__      =			tmp_path_factory.mktemp('''data''' ) / '''dataset.img.zip'''
							with zipfile.ZipFile(A				,					'''w''' ) as f:
														f.write(A				,					arcname=os.path.basename(A ) )
														f.write(A				,					arcname=os.path.basename(A ).replace('''.jpg'''				,					'''2.jpg''' ) )
							return path






@pytest.fixture(scope='''session''' )
def       _snake_case     (  A )		->					int:
							lowerCAmelCase__      =			tmp_path_factory.mktemp('''data_dir''' )

							(data_dir / "subdir").mkdir()
							with open(data_dir / '''subdir''' / '''train.txt'''				,					'''w''' ) as f:
														f.write('''foo\n''' * 10 )
							with open(data_dir / '''subdir''' / '''test.txt'''				,					'''w''' ) as f:
														f.write('''bar\n''' * 10 )
							# hidden file
							with open(data_dir / '''subdir''' / '''.test.txt'''				,					'''w''' ) as f:
														f.write('''bar\n''' * 10 )

							# hidden directory
							(data_dir / ".subdir").mkdir()
							with open(data_dir / '''.subdir''' / '''train.txt'''				,					'''w''' ) as f:
														f.write('''foo\n''' * 10 )
							with open(data_dir / '''.subdir''' / '''test.txt'''				,					'''w''' ) as f:
														f.write('''bar\n''' * 10 )

							return data_dir