



import copy
import os
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pyarrow as pa
import pyarrow.parquet as pq
import pytest

from datasets.arrow_writer import ArrowWriter, OptimizedTypedSequence, ParquetWriter, TypedSequence
from datasets.features import ArrayaD, ClassLabel, Features, Image, Value
from datasets.features.features import ArrayaDExtensionType, cast_to_python_objects
from datasets.keyhash import DuplicatedKeysError, InvalidKeyError

from .utils import require_pil






class       UpperCamelCase							(		_a      ):



	def 					__A				(  self						):
							A__					=  pa.array(TypedSequence([1, 2, 3]						)						)
							self.assertEqual(arr.type					,			pa.intaa()						)



	def 					__A				(  self						):
							with self.assertRaises(snake_case_						):
													A__					=  pa.array(TypedSequence([1, 2, 3]						)					,			type=pa.intaa()						)



	def 					__A				(  self						):
							with self.assertRaises(snake_case_						):
													A__					=  pa.array(TypedSequence([1, 2, 3]					,			try_type=Value("bool"						)					,			type=Value("int64"						)						)						)



	def 					__A				(  self						):
							A__					=  pa.array(TypedSequence([1, 2, 3]					,			type=Value("int32"						)						)						)
							self.assertEqual(arr.type					,			pa.intaa()						)



	def 					__A				(  self						):
							with self.assertRaises((TypeError, pa.lib.ArrowInvalid)						):
													A__					=  pa.array(TypedSequence(["foo", "bar"]					,			type=Value("int64"						)						)						)



	def 					__A				(  self						):
							A__					=  pa.array(TypedSequence([1, 2, 3]					,			try_type=Value("int32"						)						)						)
							self.assertEqual(arr.type					,			pa.intaa()						)



	def 					__A				(  self						):
							A__					=  pa.array(TypedSequence(["foo", "bar"]					,			try_type=Value("int64"						)						)						)
							self.assertEqual(arr.type					,			pa.string()						)



	def 					__A				(  self						):
							A__					=  pa.array(TypedSequence([[[1, 2, 3]]]					,			type=ArrayaD((1, 3)					,			"int64"						)						)						)
							self.assertEqual(arr.type					,			ArrayaDExtensionType((1, 3)					,			"int64"						)						)



	def 					__A				(  self						):
							with self.assertRaises((TypeError, pa.lib.ArrowInvalid)						):
													A__					=  pa.array(TypedSequence(["foo", "bar"]					,			type=ArrayaD((1, 3)					,			"int64"						)						)						)



	def 					__A				(  self						):
							A__					=  pa.array(TypedSequence([[[1, 2, 3]]]					,			try_type=ArrayaD((1, 3)					,			"int64"						)						)						)
							self.assertEqual(arr.type					,			ArrayaDExtensionType((1, 3)					,			"int64"						)						)



	def 					__A				(  self						):
							A__					=  pa.array(TypedSequence(["foo", "bar"]					,			try_type=ArrayaD((1, 3)					,			"int64"						)						)						)
							self.assertEqual(arr.type					,			pa.string()						)



	@require_pil
	def 					__A				(  self						):
							import PIL.Image

							A__					=  PIL.Image.fromarray(np.arange(10					,			dtype=np.uinta						).reshape(2					,			5						)						)
							with patch(
							    "datasets.arrow_writer.cast_to_python_objects"					,			side_effect=snake_case_						) as mock_cast_to_python_objects:
													A__					=  pa.array(TypedSequence([{"path": None, "bytes": b"image_bytes"}, pil_image]					,			type=Image()						)						)
													A__					=  mock_cast_to_python_objects.call_args_list[-1]
													self.assertIn("optimize_list_casting"					,			snake_case_						)
													self.assertFalse(kwargs["optimize_list_casting"]						)





def    UpperCamelCase		(  _A					: Dict					,				_A					: Any							)->  int:





						"""simple docstring"""





						A__					=  pa.BufferReader(_lowerCAmelCase							) if isinstance(_lowerCAmelCase					,				pa.Buffer							) else pa.memory_map(_lowerCAmelCase							)
						A__					=  pa.ipc.open_stream(_lowerCAmelCase							)
						A__					=  f.read_all()
						assert len(pa_table.to_batches()							) == expected_num_chunks
						assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}
						del pa_table
@pytest.mark.parametrize("writer_batch_size"					,				[None, 1, 10]							)
@pytest.mark.parametrize(
    "fields"					,				[None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}]							)
def    UpperCamelCase		(  _A					: Dict					,				_A					: str							)->  Optional[Any]:





						"""simple docstring"""





						A__					=  pa.BufferOutputStream()
						A__					=  pa.schema(_lowerCAmelCase							) if fields else None
						with ArrowWriter(stream=_lowerCAmelCase					,				schema=_lowerCAmelCase					,				writer_batch_size=_lowerCAmelCase							) as writer:
												writer.write({"col_1": "foo", "col_2": 1}							)
												writer.write({"col_1": "bar", "col_2": 2}							)
												A__					=  writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						if not fields:
												A__					=  {"""col_1""": pa.string(), """col_2""": pa.intaa()}
						assert writer._schema == pa.schema(_lowerCAmelCase					,				metadata=writer._schema.metadata							)
						_check_output(output.getvalue()					,				expected_num_chunks=num_examples if writer_batch_size == 1 else 1							)
def    UpperCamelCase		(  )->  List[Any]:





						"""simple docstring"""





						A__					=  pa.BufferOutputStream()
						A__					=  Features({"labels": ClassLabel(names=["neg", "pos"]							)}							)
						with ArrowWriter(stream=_lowerCAmelCase					,				features=_lowerCAmelCase							) as writer:
												writer.write({"labels": 0}							)
												writer.write({"labels": 1}							)
												A__					=  writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						assert writer._schema == features.arrow_schema
						assert writer._schema.metadata == features.arrow_schema.metadata
						A__					=  pa.BufferReader(output.getvalue()							)
						A__					=  pa.ipc.open_stream(_lowerCAmelCase							)
						A__					=  f.read_all()
						A__					=  pa_table.schema
						assert pa_table.num_rows == 2
						assert schema == features.arrow_schema
						assert schema.metadata == features.arrow_schema.metadata
						assert features == Features.from_arrow_schema(_lowerCAmelCase							)
@pytest.mark.parametrize("writer_batch_size"					,				[None, 1, 10]							)
def    UpperCamelCase		(  _A					: Optional[int]							)->  Any:





						"""simple docstring"""





						A__					=  pa.BufferOutputStream()
						with ArrowWriter(
						    stream=_lowerCAmelCase					,				writer_batch_size=_lowerCAmelCase					,				hash_salt="split_name"					,				check_duplicates=_lowerCAmelCase					,				) as writer:
												with pytest.raises(_lowerCAmelCase							):
																		writer.write({"col_1": "foo", "col_2": 1}					,				key=[1, 2]							)
																		A__					=  writer.finalize()
@pytest.mark.parametrize("writer_batch_size"					,				[None, 2, 10]							)
def    UpperCamelCase		(  _A					: Tuple							)->  List[str]:





						"""simple docstring"""





						A__					=  pa.BufferOutputStream()
						with ArrowWriter(
						    stream=_lowerCAmelCase					,				writer_batch_size=_lowerCAmelCase					,				hash_salt="split_name"					,				check_duplicates=_lowerCAmelCase					,				) as writer:
												with pytest.raises(_lowerCAmelCase							):
																		writer.write({"col_1": "foo", "col_2": 1}					,				key=10							)
																		writer.write({"col_1": "bar", "col_2": 2}					,				key=10							)
																		A__					=  writer.finalize()
@pytest.mark.parametrize("writer_batch_size"					,				[None, 2, 10]							)
def    UpperCamelCase		(  _A					: Dict							)->  int:





						"""simple docstring"""





						A__					=  pa.BufferOutputStream()
						with ArrowWriter(
						    stream=_lowerCAmelCase					,				writer_batch_size=_lowerCAmelCase					,				hash_salt="split_name"					,				check_duplicates=_lowerCAmelCase					,				) as writer:
												writer.write({"col_1": "foo", "col_2": 1}					,				key=1							)
												writer.write({"col_1": "bar", "col_2": 2}					,				key=2							)
												A__					=  writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						_check_output(output.getvalue()					,				expected_num_chunks=num_examples if writer_batch_size == 1 else 1							)
@pytest.mark.parametrize("writer_batch_size"					,				[None, 1, 10]							)
@pytest.mark.parametrize(
    "fields"					,				[None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}]							)
def    UpperCamelCase		(  _A					: List[Any]					,				_A					: Dict							)->  Tuple:





						"""simple docstring"""





						A__					=  pa.BufferOutputStream()
						A__					=  pa.schema(_lowerCAmelCase							) if fields else None
						with ArrowWriter(stream=_lowerCAmelCase					,				schema=_lowerCAmelCase					,				writer_batch_size=_lowerCAmelCase							) as writer:
												writer.write_batch({"col_1": ["foo", "bar"], "col_2": [1, 2]}							)
												writer.write_batch({"col_1": [], "col_2": []}							)
												A__					=  writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						if not fields:
												A__					=  {"""col_1""": pa.string(), """col_2""": pa.intaa()}
						assert writer._schema == pa.schema(_lowerCAmelCase					,				metadata=writer._schema.metadata							)
						_check_output(output.getvalue()					,				expected_num_chunks=num_examples if writer_batch_size == 1 else 1							)
@pytest.mark.parametrize("writer_batch_size"					,				[None, 1, 10]							)
@pytest.mark.parametrize(
    "fields"					,				[None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}]							)
def    UpperCamelCase		(  _A					: Dict					,				_A					: List[Any]							)->  List[Any]:





						"""simple docstring"""





						A__					=  pa.BufferOutputStream()
						A__					=  pa.schema(_lowerCAmelCase							) if fields else None
						with ArrowWriter(stream=_lowerCAmelCase					,				schema=_lowerCAmelCase					,				writer_batch_size=_lowerCAmelCase							) as writer:
												writer.write_table(pa.Table.from_pydict({"col_1": ["foo", "bar"], "col_2": [1, 2]}							)							)
												A__					=  writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						if not fields:
												A__					=  {"""col_1""": pa.string(), """col_2""": pa.intaa()}
						assert writer._schema == pa.schema(_lowerCAmelCase					,				metadata=writer._schema.metadata							)
						_check_output(output.getvalue()					,				expected_num_chunks=num_examples if writer_batch_size == 1 else 1							)
@pytest.mark.parametrize("writer_batch_size"					,				[None, 1, 10]							)
@pytest.mark.parametrize(
    "fields"					,				[None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}]							)
def    UpperCamelCase		(  _A					: int					,				_A					: List[Any]							)->  str:





						"""simple docstring"""





						A__					=  pa.BufferOutputStream()
						A__					=  pa.schema(_lowerCAmelCase							) if fields else None
						with ArrowWriter(stream=_lowerCAmelCase					,				schema=_lowerCAmelCase					,				writer_batch_size=_lowerCAmelCase							) as writer:
												writer.write_row(pa.Table.from_pydict({"col_1": ["foo"], "col_2": [1]}							)							)
												writer.write_row(pa.Table.from_pydict({"col_1": ["bar"], "col_2": [2]}							)							)
												A__					=  writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						if not fields:
												A__					=  {"""col_1""": pa.string(), """col_2""": pa.intaa()}
						assert writer._schema == pa.schema(_lowerCAmelCase					,				metadata=writer._schema.metadata							)
						_check_output(output.getvalue()					,				expected_num_chunks=num_examples if writer_batch_size == 1 else 1							)
def    UpperCamelCase		(  )->  List[str]:





						"""simple docstring"""





						with tempfile.TemporaryDirectory() as tmp_dir:
												A__					=  {"""col_1""": pa.string(), """col_2""": pa.intaa()}
												A__					=  os.path.join(_lowerCAmelCase					,				"test.arrow"							)
												with ArrowWriter(path=_lowerCAmelCase					,				schema=pa.schema(_lowerCAmelCase							)							) as writer:
																		writer.write_batch({"col_1": ["foo", "bar"], "col_2": [1, 2]}							)
																		A__					=  writer.finalize()
												assert num_examples == 2
												assert num_bytes > 0
												assert writer._schema == pa.schema(_lowerCAmelCase					,				metadata=writer._schema.metadata							)
												_check_output(_lowerCAmelCase					,				1							)
def    UpperCamelCase		(  _A					: Optional[int]							)->  List[str]:





						"""simple docstring"""





						if pa.types.is_list(_lowerCAmelCase							):
												return get_base_dtype(arr_type.value_type							)
						else:
												return arr_type
def    UpperCamelCase		(  _A					: List[str]					,				_A					: Any							)->  List[str]:





						"""simple docstring"""





						if isinstance(lst[0]					,				_lowerCAmelCase							):
												change_first_primitive_element_in_list(lst[0]					,				_lowerCAmelCase							)
						else:
												A__					=  value
@pytest.mark.parametrize("optimized_int_type, expected_dtype"					,				[(None, pa.intaa()), (Value("int32"							), pa.intaa())]							)
@pytest.mark.parametrize("sequence"					,				[[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]]							)
def    UpperCamelCase		(  _A					: str					,				_A					: Optional[int]					,				_A					: Optional[int]							)->  Optional[int]:





						"""simple docstring"""





						A__					=  pa.array(TypedSequence(_lowerCAmelCase					,				optimized_int_type=_lowerCAmelCase							)							)
						assert get_base_dtype(arr.type							) == expected_dtype
@pytest.mark.parametrize(
    "col, expected_dtype"					,				[
        ("attention_mask", pa.inta()),
        ("special_tokens_mask", pa.inta()),
        ("token_type_ids", pa.inta()),
        ("input_ids", pa.intaa()),
        ("other", pa.intaa()),
    ]					,				)
@pytest.mark.parametrize("sequence"					,				[[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]]							)
def    UpperCamelCase		(  _A					: List[Any]					,				_A					: List[str]					,				_A					: List[Any]							)->  Optional[int]:





						"""simple docstring"""





						A__					=  pa.array(OptimizedTypedSequence(_lowerCAmelCase					,				col=_lowerCAmelCase							)							)
						assert get_base_dtype(arr.type							) == expected_dtype

						# not in range
						if col != "other":
												# avoids errors due to in-place modifications
												A__					=  copy.deepcopy(_lowerCAmelCase							)
												A__					=  np.iinfo(expected_dtype.to_pandas_dtype()							).max + 1
												change_first_primitive_element_in_list(_lowerCAmelCase					,				_lowerCAmelCase							)
												A__					=  pa.array(OptimizedTypedSequence(_lowerCAmelCase					,				col=_lowerCAmelCase							)							)
												assert get_base_dtype(arr.type							) == pa.intaa()
@pytest.mark.parametrize("raise_exception"					,				[False, True]							)
def    UpperCamelCase		(  _A					: str					,				_A					: Union[str, Any]							)->  Optional[int]:





						"""simple docstring"""





						A__					=  str(tmp_path / "dataset-train.arrow"							)
						try:
												with ArrowWriter(path=_lowerCAmelCase							) as writer:
																		if raise_exception:
																								raise pa.lib.ArrowInvalid()
																		else:
																								writer.stream.close()
						except pa.lib.ArrowInvalid:
												pass
						finally:
												assert writer.stream.closed
def    UpperCamelCase		(  _A					: Optional[int]							)->  List[Any]:





						"""simple docstring"""





						A__					=  """mock://dataset-train.arrow"""
						with ArrowWriter(path=_lowerCAmelCase					,				storage_options=mockfs.storage_options							) as writer:
												assert isinstance(writer._fs					,				type(_lowerCAmelCase							)							)
												assert writer._fs.storage_options == mockfs.storage_options
												writer.write({"col_1": "foo", "col_2": 1}							)
												writer.write({"col_1": "bar", "col_2": 2}							)
												A__					=  writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						assert mockfs.exists(_lowerCAmelCase							)
def    UpperCamelCase		(  )->  List[str]:





						"""simple docstring"""





						A__					=  pa.BufferOutputStream()
						with ParquetWriter(stream=_lowerCAmelCase							) as writer:
												writer.write({"col_1": "foo", "col_2": 1}							)
												writer.write({"col_1": "bar", "col_2": 2}							)
												A__					=  writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						A__					=  pa.BufferReader(output.getvalue()							)
						A__					=  pq.read_table(_lowerCAmelCase							)
						assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}
@require_pil
@pytest.mark.parametrize("embed_local_files"					,				[False, True]							)
def    UpperCamelCase		(  _A					: str					,				_A					: List[Any]							)->  List[str]:





						"""simple docstring"""





						import PIL.Image

						A__					=  str(tmp_path / "test_image_rgb.jpg"							)
						PIL.Image.fromarray(np.zeros((5, 5)					,				dtype=np.uinta							)							).save(_lowerCAmelCase					,				format="png"							)
						A__					=  pa.BufferOutputStream()
						with ParquetWriter(
						    stream=_lowerCAmelCase					,				features=Features({"image": Image()}							)					,				embed_local_files=_lowerCAmelCase							) as writer:
												writer.write({"image": image_path}							)
												writer.finalize()
						A__					=  pa.BufferReader(output.getvalue()							)
						A__					=  pq.read_table(_lowerCAmelCase							)
						A__					=  pa_table.to_pydict()
						if embed_local_files:
												assert isinstance(out["image"][0]["path"]					,				_lowerCAmelCase							)
												with open(_lowerCAmelCase					,				"rb"							) as f:
																		assert out["image"][0]["bytes"] == f.read()
						else:
												assert out["image"][0]["path"] == image_path
												assert out["image"][0]["bytes"] is None
def    UpperCamelCase		(  )->  Tuple:





						"""simple docstring"""





						A__					=  pa.schema([pa.field("col_1"					,				pa.string()					,				nullable=_lowerCAmelCase							)]							)
						A__					=  pa.BufferOutputStream()
						with ArrowWriter(stream=_lowerCAmelCase							) as writer:
												writer._build_writer(inferred_schema=_lowerCAmelCase							)
												assert writer._schema == pa.schema([pa.field("col_1"					,				pa.string()							)]							)










'''simple docstring'''





from ...configuration_utils import PretrainedConfig
from ...utils import logging
from ...utils.backbone_utils import BackboneConfigMixin, get_aligned_output_features_output_indices


__a  			=						logging.get_logger(__name__)

__a  			=						{
    "shi-labs/dinat-mini-in1k-224": "https://huggingface.co/shi-labs/dinat-mini-in1k-224/resolve/main/config.json",
    # See all Dinat models at https://huggingface.co/models?filter=dinat
}







class        UpperCAmelCase_			(			_a						,						_a      ):
    """simple docstring"""







    lowercase   =						"dinat"

    lowercase   =						{
        "num_attention_heads": "num_heads",
        "num_hidden_layers": "num_layers",
    }




    def __init__(       self   :	Optional[int]					,	snake_case_   :	Optional[Any]=4					,	snake_case_   :	List[str]=3					,	snake_case_   :	Tuple=64					,	snake_case_   :	List[Any]=[3, 4, 6, 5]					,	snake_case_   :	List[Any]=[2, 4, 8, 16]					,	snake_case_   :	List[str]=7					,	snake_case_   :	List[str]=[[1, 8, 1], [1, 4, 1, 4], [1, 2, 1, 2, 1, 2], [1, 1, 1, 1, 1]]					,	snake_case_   :	List[Any]=3.0					,	snake_case_   :	Dict=True					,	snake_case_   :	int=0.0					,	snake_case_   :	Optional[int]=0.0					,	snake_case_   :	str=0.1					,	snake_case_   :	Optional[Any]="gelu"					,	snake_case_   :	Union[str, Any]=0.02					,	snake_case_   :	Optional[int]=1E-5					,	snake_case_   :	str=0.0					,	snake_case_   :	Dict=None					,	snake_case_   :	Dict=None					,	**snake_case_   :	Dict					,	):
          super().__init__(**snake_case_		)

          snake_case__    :       Optional[Any]       		= patch_size
          snake_case__    :       Union[str, Any]       		= num_channels
          snake_case__    :       int       		= embed_dim
          snake_case__    :       Dict       		= depths
          snake_case__    :       Dict       		= len(snake_case_		)
          snake_case__    :       Optional[int]       		= num_heads
          snake_case__    :       Any       		= kernel_size
          snake_case__    :       str       		= dilations
          snake_case__    :       Any       		= mlp_ratio
          snake_case__    :       int       		= qkv_bias
          snake_case__    :       int       		= hidden_dropout_prob
          snake_case__    :       int       		= attention_probs_dropout_prob
          snake_case__    :       Optional[Any]       		= drop_path_rate
          snake_case__    :       int       		= hidden_act
          snake_case__    :       Tuple       		= layer_norm_eps
          snake_case__    :       Any       		= initializer_range
          # we set the hidden_size attribute in order to make Dinat work with VisionEncoderDecoderModel
          # this indicates the channel dimension after the last stage of the model
          snake_case__    :       Optional[Any]       		= int(embed_dim * 2 ** (len(snake_case_		) - 1)		)
          snake_case__    :       int       		= layer_scale_init_value
          snake_case__    :       Dict       		= ["""stem"""] + [f"stage{idx}" for idx in range(1					,	len(snake_case_		) + 1		)]
          snake_case__		,							snake_case__    :       Tuple       		= get_aligned_output_features_output_indices(
              out_features=snake_case_					,	out_indices=snake_case_					,	stage_names=self.stage_names		)
