






import copy
import os
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pyarrow as pa
import pyarrow.parquet as pq
import pytest

from datasets.arrow_writer import ArrowWriter, OptimizedTypedSequence, ParquetWriter, TypedSequence
from datasets.features import ArrayaD, ClassLabel, Features, Image, Value
from datasets.features.features import ArrayaDExtensionType, cast_to_python_objects
from datasets.keyhash import DuplicatedKeysError, InvalidKeyError

from .utils import require_pil
class  lowerCAmelCase_       (				snake_case__					):




							def 						__snake_case					(   self     :   str	):
											lowerCAmelCase__      			=						pa.array(TypedSequence([1, 2, 3]	)	)
											self.assertEqual(arr.type ,							pa.intaa()	)




							def 						__snake_case					(   self     :   Optional[int]	):
											with self.assertRaises(SCREAMING_SNAKE_CASE_	):
															lowerCAmelCase__      			=						pa.array(TypedSequence([1, 2, 3]	) ,							type=pa.intaa()	)




							def 						__snake_case					(   self     :   str	):
											with self.assertRaises(SCREAMING_SNAKE_CASE_	):
															lowerCAmelCase__      			=						pa.array(TypedSequence([1, 2, 3] ,							try_type=Value('''bool'''	) ,							type=Value('''int64'''	)	)	)




							def 						__snake_case					(   self     :   List[str]	):
											lowerCAmelCase__      			=						pa.array(TypedSequence([1, 2, 3] ,							type=Value('''int32'''	)	)	)
											self.assertEqual(arr.type ,							pa.intaa()	)




							def 						__snake_case					(   self     :   Dict	):
											with self.assertRaises((TypeError, pa.lib.ArrowInvalid)	):
															lowerCAmelCase__      			=						pa.array(TypedSequence(['''foo''', '''bar'''] ,							type=Value('''int64'''	)	)	)




							def 						__snake_case					(   self     :   List[Any]	):
											lowerCAmelCase__      			=						pa.array(TypedSequence([1, 2, 3] ,							try_type=Value('''int32'''	)	)	)
											self.assertEqual(arr.type ,							pa.intaa()	)




							def 						__snake_case					(   self     :   str	):
											lowerCAmelCase__      			=						pa.array(TypedSequence(['''foo''', '''bar'''] ,							try_type=Value('''int64'''	)	)	)
											self.assertEqual(arr.type ,							pa.string()	)




							def 						__snake_case					(   self     :   Dict	):
											lowerCAmelCase__      			=						pa.array(TypedSequence([[[1, 2, 3]]] ,							type=ArrayaD((1, 3) ,							'''int64'''	)	)	)
											self.assertEqual(arr.type ,							ArrayaDExtensionType((1, 3) ,							'''int64'''	)	)




							def 						__snake_case					(   self     :   Tuple	):
											with self.assertRaises((TypeError, pa.lib.ArrowInvalid)	):
															lowerCAmelCase__      			=						pa.array(TypedSequence(['''foo''', '''bar'''] ,							type=ArrayaD((1, 3) ,							'''int64'''	)	)	)




							def 						__snake_case					(   self     :   Union[str, Any]	):
											lowerCAmelCase__      			=						pa.array(TypedSequence([[[1, 2, 3]]] ,							try_type=ArrayaD((1, 3) ,							'''int64'''	)	)	)
											self.assertEqual(arr.type ,							ArrayaDExtensionType((1, 3) ,							'''int64'''	)	)




							def 						__snake_case					(   self     :   Optional[int]	):
											lowerCAmelCase__      			=						pa.array(TypedSequence(['''foo''', '''bar'''] ,							try_type=ArrayaD((1, 3) ,							'''int64'''	)	)	)
											self.assertEqual(arr.type ,							pa.string()	)




							@require_pil
							def 						__snake_case					(   self     :   Optional[int]	):
											import PIL.Image

											lowerCAmelCase__      			=						PIL.Image.fromarray(np.arange(10 ,							dtype=np.uinta	).reshape(2 ,							5	)	)
											with patch(
											    '''datasets.arrow_writer.cast_to_python_objects''' ,							side_effect=SCREAMING_SNAKE_CASE_	) as mock_cast_to_python_objects:
															lowerCAmelCase__      			=						pa.array(TypedSequence([{'''path''': None, '''bytes''': b'''image_bytes'''}, pil_image] ,							type=Image()	)	)
															lowerCAmelCase__			,		lowerCAmelCase__      			=						mock_cast_to_python_objects.call_args_list[-1]
															self.assertIn('''optimize_list_casting''' ,							SCREAMING_SNAKE_CASE_	)
															self.assertFalse(kwargs['''optimize_list_casting''']	)





def 	lowerCAmelCase_		(lowercase__						:       int     ,					lowercase__						:       int       )		->    List[Any]:


				'''simple docstring'''




				lowerCAmelCase__      			=						pa.BufferReader(lowercase__       ) if isinstance(lowercase__     ,					pa.Buffer       ) else pa.memory_map(lowercase__       )
				lowerCAmelCase__      			=						pa.ipc.open_stream(lowercase__       )
				lowerCAmelCase__      			=						f.read_all()
				assert len(pa_table.to_batches()       ) == expected_num_chunks
				assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}
				del pa_table





@pytest.mark.parametrize('''writer_batch_size'''     ,					[None, 1, 10]       )
@pytest.mark.parametrize(
    '''fields'''     ,					[None, {'''col_1''': pa.string(), '''col_2''': pa.intaa()}, {'''col_1''': pa.string(), '''col_2''': pa.intaa()}]       )
def 	lowerCAmelCase_		(lowercase__						:       Tuple     ,					lowercase__						:       Optional[Any]       )		->    str:


				'''simple docstring'''




				lowerCAmelCase__      			=						pa.BufferOutputStream()
				lowerCAmelCase__      			=						pa.schema(lowercase__       ) if fields else None
				with ArrowWriter(stream=lowercase__     ,					schema=lowercase__     ,					writer_batch_size=lowercase__       ) as writer:
								writer.write({'''col_1''': '''foo''', '''col_2''': 1}       )
								writer.write({'''col_1''': '''bar''', '''col_2''': 2}       )
								lowerCAmelCase__			,		lowerCAmelCase__      			=						writer.finalize()
				assert num_examples == 2
				assert num_bytes > 0
				if not fields:
								lowerCAmelCase__      			=						{'''col_1''': pa.string(), '''col_2''': pa.intaa()}
				assert writer._schema == pa.schema(lowercase__     ,					metadata=writer._schema.metadata       )
				_check_output(output.getvalue()     ,					expected_num_chunks=num_examples if writer_batch_size == 1 else 1       )





def 	lowerCAmelCase_		()		->    Optional[int]:


				'''simple docstring'''




				lowerCAmelCase__      			=						pa.BufferOutputStream()
				lowerCAmelCase__      			=						Features({'''labels''': ClassLabel(names=['''neg''', '''pos''']       )}       )
				with ArrowWriter(stream=lowercase__     ,					features=lowercase__       ) as writer:
								writer.write({'''labels''': 0}       )
								writer.write({'''labels''': 1}       )
								lowerCAmelCase__			,		lowerCAmelCase__      			=						writer.finalize()
				assert num_examples == 2
				assert num_bytes > 0
				assert writer._schema == features.arrow_schema
				assert writer._schema.metadata == features.arrow_schema.metadata
				lowerCAmelCase__      			=						pa.BufferReader(output.getvalue()       )
				lowerCAmelCase__      			=						pa.ipc.open_stream(lowercase__       )
				lowerCAmelCase__      			=						f.read_all()
				lowerCAmelCase__      			=						pa_table.schema
				assert pa_table.num_rows == 2
				assert schema == features.arrow_schema
				assert schema.metadata == features.arrow_schema.metadata
				assert features == Features.from_arrow_schema(lowercase__       )





@pytest.mark.parametrize('''writer_batch_size'''     ,					[None, 1, 10]       )
def 	lowerCAmelCase_		(lowercase__						:       Tuple       )		->    List[Any]:


				'''simple docstring'''




				lowerCAmelCase__      			=						pa.BufferOutputStream()
				with ArrowWriter(
				    stream=lowercase__     ,					writer_batch_size=lowercase__     ,					hash_salt='''split_name'''     ,					check_duplicates=lowercase__     ,					) as writer:
								with pytest.raises(lowercase__       ):
												writer.write({'''col_1''': '''foo''', '''col_2''': 1}     ,					key=[1, 2]       )
												lowerCAmelCase__			,		lowerCAmelCase__      			=						writer.finalize()





@pytest.mark.parametrize('''writer_batch_size'''     ,					[None, 2, 10]       )
def 	lowerCAmelCase_		(lowercase__						:       List[Any]       )		->    str:


				'''simple docstring'''




				lowerCAmelCase__      			=						pa.BufferOutputStream()
				with ArrowWriter(
				    stream=lowercase__     ,					writer_batch_size=lowercase__     ,					hash_salt='''split_name'''     ,					check_duplicates=lowercase__     ,					) as writer:
								with pytest.raises(lowercase__       ):
												writer.write({'''col_1''': '''foo''', '''col_2''': 1}     ,					key=10       )
												writer.write({'''col_1''': '''bar''', '''col_2''': 2}     ,					key=10       )
												lowerCAmelCase__			,		lowerCAmelCase__      			=						writer.finalize()





@pytest.mark.parametrize('''writer_batch_size'''     ,					[None, 2, 10]       )
def 	lowerCAmelCase_		(lowercase__						:       Tuple       )		->    Union[str, Any]:


				'''simple docstring'''




				lowerCAmelCase__      			=						pa.BufferOutputStream()
				with ArrowWriter(
				    stream=lowercase__     ,					writer_batch_size=lowercase__     ,					hash_salt='''split_name'''     ,					check_duplicates=lowercase__     ,					) as writer:
								writer.write({'''col_1''': '''foo''', '''col_2''': 1}     ,					key=1       )
								writer.write({'''col_1''': '''bar''', '''col_2''': 2}     ,					key=2       )
								lowerCAmelCase__			,		lowerCAmelCase__      			=						writer.finalize()
				assert num_examples == 2
				assert num_bytes > 0
				_check_output(output.getvalue()     ,					expected_num_chunks=num_examples if writer_batch_size == 1 else 1       )





@pytest.mark.parametrize('''writer_batch_size'''     ,					[None, 1, 10]       )
@pytest.mark.parametrize(
    '''fields'''     ,					[None, {'''col_1''': pa.string(), '''col_2''': pa.intaa()}, {'''col_1''': pa.string(), '''col_2''': pa.intaa()}]       )
def 	lowerCAmelCase_		(lowercase__						:       Tuple     ,					lowercase__						:       int       )		->    Optional[int]:


				'''simple docstring'''




				lowerCAmelCase__      			=						pa.BufferOutputStream()
				lowerCAmelCase__      			=						pa.schema(lowercase__       ) if fields else None
				with ArrowWriter(stream=lowercase__     ,					schema=lowercase__     ,					writer_batch_size=lowercase__       ) as writer:
								writer.write_batch({'''col_1''': ['''foo''', '''bar'''], '''col_2''': [1, 2]}       )
								writer.write_batch({'''col_1''': [], '''col_2''': []}       )
								lowerCAmelCase__			,		lowerCAmelCase__      			=						writer.finalize()
				assert num_examples == 2
				assert num_bytes > 0
				if not fields:
								lowerCAmelCase__      			=						{'''col_1''': pa.string(), '''col_2''': pa.intaa()}
				assert writer._schema == pa.schema(lowercase__     ,					metadata=writer._schema.metadata       )
				_check_output(output.getvalue()     ,					expected_num_chunks=num_examples if writer_batch_size == 1 else 1       )





@pytest.mark.parametrize('''writer_batch_size'''     ,					[None, 1, 10]       )
@pytest.mark.parametrize(
    '''fields'''     ,					[None, {'''col_1''': pa.string(), '''col_2''': pa.intaa()}, {'''col_1''': pa.string(), '''col_2''': pa.intaa()}]       )
def 	lowerCAmelCase_		(lowercase__						:       Optional[Any]     ,					lowercase__						:       int       )		->    List[Any]:


				'''simple docstring'''




				lowerCAmelCase__      			=						pa.BufferOutputStream()
				lowerCAmelCase__      			=						pa.schema(lowercase__       ) if fields else None
				with ArrowWriter(stream=lowercase__     ,					schema=lowercase__     ,					writer_batch_size=lowercase__       ) as writer:
								writer.write_table(pa.Table.from_pydict({'''col_1''': ['''foo''', '''bar'''], '''col_2''': [1, 2]}       )       )
								lowerCAmelCase__			,		lowerCAmelCase__      			=						writer.finalize()
				assert num_examples == 2
				assert num_bytes > 0
				if not fields:
								lowerCAmelCase__      			=						{'''col_1''': pa.string(), '''col_2''': pa.intaa()}
				assert writer._schema == pa.schema(lowercase__     ,					metadata=writer._schema.metadata       )
				_check_output(output.getvalue()     ,					expected_num_chunks=num_examples if writer_batch_size == 1 else 1       )





@pytest.mark.parametrize('''writer_batch_size'''     ,					[None, 1, 10]       )
@pytest.mark.parametrize(
    '''fields'''     ,					[None, {'''col_1''': pa.string(), '''col_2''': pa.intaa()}, {'''col_1''': pa.string(), '''col_2''': pa.intaa()}]       )
def 	lowerCAmelCase_		(lowercase__						:       Union[str, Any]     ,					lowercase__						:       int       )		->    int:


				'''simple docstring'''




				lowerCAmelCase__      			=						pa.BufferOutputStream()
				lowerCAmelCase__      			=						pa.schema(lowercase__       ) if fields else None
				with ArrowWriter(stream=lowercase__     ,					schema=lowercase__     ,					writer_batch_size=lowercase__       ) as writer:
								writer.write_row(pa.Table.from_pydict({'''col_1''': ['''foo'''], '''col_2''': [1]}       )       )
								writer.write_row(pa.Table.from_pydict({'''col_1''': ['''bar'''], '''col_2''': [2]}       )       )
								lowerCAmelCase__			,		lowerCAmelCase__      			=						writer.finalize()
				assert num_examples == 2
				assert num_bytes > 0
				if not fields:
								lowerCAmelCase__      			=						{'''col_1''': pa.string(), '''col_2''': pa.intaa()}
				assert writer._schema == pa.schema(lowercase__     ,					metadata=writer._schema.metadata       )
				_check_output(output.getvalue()     ,					expected_num_chunks=num_examples if writer_batch_size == 1 else 1       )





def 	lowerCAmelCase_		()		->    Dict:


				'''simple docstring'''




				with tempfile.TemporaryDirectory() as tmp_dir:
								lowerCAmelCase__      			=						{'''col_1''': pa.string(), '''col_2''': pa.intaa()}
								lowerCAmelCase__      			=						os.path.join(lowercase__     ,					'''test.arrow'''       )
								with ArrowWriter(path=lowercase__     ,					schema=pa.schema(lowercase__       )       ) as writer:
												writer.write_batch({'''col_1''': ['''foo''', '''bar'''], '''col_2''': [1, 2]}       )
												lowerCAmelCase__			,		lowerCAmelCase__      			=						writer.finalize()
								assert num_examples == 2
								assert num_bytes > 0
								assert writer._schema == pa.schema(lowercase__     ,					metadata=writer._schema.metadata       )
								_check_output(lowercase__     ,					1       )





def 	lowerCAmelCase_		(lowercase__						:       Dict       )		->    Tuple:


				'''simple docstring'''




				if pa.types.is_list(lowercase__       ):
								return get_base_dtype(arr_type.value_type       )
				else:
								return arr_type





def 	lowerCAmelCase_		(lowercase__						:       int     ,					lowercase__						:       Dict       )		->    Tuple:


				'''simple docstring'''




				if isinstance(lst[0]     ,					lowercase__       ):
								change_first_primitive_element_in_list(lst[0]     ,					lowercase__       )
				else:
								lowerCAmelCase__      			=						value





@pytest.mark.parametrize('''optimized_int_type, expected_dtype'''     ,					[(None, pa.intaa()), (Value('''int32'''       ), pa.intaa())]       )
@pytest.mark.parametrize('''sequence'''     ,					[[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]]       )
def 	lowerCAmelCase_		(lowercase__						:       Union[str, Any]     ,					lowercase__						:       str     ,					lowercase__						:       Tuple       )		->    Tuple:


				'''simple docstring'''




				lowerCAmelCase__      			=						pa.array(TypedSequence(lowercase__     ,					optimized_int_type=lowercase__       )       )
				assert get_base_dtype(arr.type       ) == expected_dtype





@pytest.mark.parametrize(
    '''col, expected_dtype'''     ,					[
        ('''attention_mask''', pa.inta()),
        ('''special_tokens_mask''', pa.inta()),
        ('''token_type_ids''', pa.inta()),
        ('''input_ids''', pa.intaa()),
        ('''other''', pa.intaa()),
    ]     ,					)
@pytest.mark.parametrize('''sequence'''     ,					[[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]]       )
def 	lowerCAmelCase_		(lowercase__						:       Tuple     ,					lowercase__						:       str     ,					lowercase__						:       List[Any]       )		->    Optional[Any]:


				'''simple docstring'''




				lowerCAmelCase__      			=						pa.array(OptimizedTypedSequence(lowercase__     ,					col=lowercase__       )       )
				assert get_base_dtype(arr.type       ) == expected_dtype

				# not in range
				if col != "other":
								# avoids errors due to in-place modifications
								lowerCAmelCase__      			=						copy.deepcopy(lowercase__       )
								lowerCAmelCase__      			=						np.iinfo(expected_dtype.to_pandas_dtype()       ).max + 1
								change_first_primitive_element_in_list(lowercase__     ,					lowercase__       )
								lowerCAmelCase__      			=						pa.array(OptimizedTypedSequence(lowercase__     ,					col=lowercase__       )       )
								assert get_base_dtype(arr.type       ) == pa.intaa()





@pytest.mark.parametrize('''raise_exception'''     ,					[False, True]       )
def 	lowerCAmelCase_		(lowercase__						:       Tuple     ,					lowercase__						:       Optional[int]       )		->    Optional[int]:


				'''simple docstring'''




				lowerCAmelCase__      			=						str(tmp_path / '''dataset-train.arrow'''       )
				try:
								with ArrowWriter(path=lowercase__       ) as writer:
												if raise_exception:
																raise pa.lib.ArrowInvalid()
												else:
																writer.stream.close()
				except pa.lib.ArrowInvalid:
								pass
				finally:
								assert writer.stream.closed





def 	lowerCAmelCase_		(lowercase__						:       str       )		->    Dict:


				'''simple docstring'''




				lowerCAmelCase__      			=						'''mock://dataset-train.arrow'''
				with ArrowWriter(path=lowercase__     ,					storage_options=mockfs.storage_options       ) as writer:
								assert isinstance(writer._fs     ,					type(lowercase__       )       )
								assert writer._fs.storage_options == mockfs.storage_options
								writer.write({'''col_1''': '''foo''', '''col_2''': 1}       )
								writer.write({'''col_1''': '''bar''', '''col_2''': 2}       )
								lowerCAmelCase__			,		lowerCAmelCase__      			=						writer.finalize()
				assert num_examples == 2
				assert num_bytes > 0
				assert mockfs.exists(lowercase__       )





def 	lowerCAmelCase_		()		->    Optional[int]:


				'''simple docstring'''




				lowerCAmelCase__      			=						pa.BufferOutputStream()
				with ParquetWriter(stream=lowercase__       ) as writer:
								writer.write({'''col_1''': '''foo''', '''col_2''': 1}       )
								writer.write({'''col_1''': '''bar''', '''col_2''': 2}       )
								lowerCAmelCase__			,		lowerCAmelCase__      			=						writer.finalize()
				assert num_examples == 2
				assert num_bytes > 0
				lowerCAmelCase__      			=						pa.BufferReader(output.getvalue()       )
				lowerCAmelCase__      			=						pq.read_table(lowercase__       )
				assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}





@require_pil
@pytest.mark.parametrize('''embed_local_files'''     ,					[False, True]       )
def 	lowerCAmelCase_		(lowercase__						:       List[str]     ,					lowercase__						:       List[Any]       )		->    Optional[Any]:


				'''simple docstring'''




				import PIL.Image

				lowerCAmelCase__      			=						str(tmp_path / '''test_image_rgb.jpg'''       )
				PIL.Image.fromarray(np.zeros((5, 5)     ,					dtype=np.uinta       )       ).save(lowercase__     ,					format='''png'''       )
				lowerCAmelCase__      			=						pa.BufferOutputStream()
				with ParquetWriter(
				    stream=lowercase__     ,					features=Features({'''image''': Image()}       )     ,					embed_local_files=lowercase__       ) as writer:
								writer.write({'''image''': image_path}       )
								writer.finalize()
				lowerCAmelCase__      			=						pa.BufferReader(output.getvalue()       )
				lowerCAmelCase__      			=						pq.read_table(lowercase__       )
				lowerCAmelCase__      			=						pa_table.to_pydict()
				if embed_local_files:
								assert isinstance(out['''image'''][0]['''path''']     ,					lowercase__       )
								with open(lowercase__     ,					'''rb'''       ) as f:
												assert out["image"][0]["bytes"] == f.read()
				else:
								assert out["image"][0]["path"] == image_path
								assert out["image"][0]["bytes"] is None





def 	lowerCAmelCase_		()		->    Optional[int]:


				'''simple docstring'''




				lowerCAmelCase__      			=						pa.schema([pa.field('''col_1'''     ,					pa.string()     ,					nullable=lowercase__       )]       )
				lowerCAmelCase__      			=						pa.BufferOutputStream()
				with ArrowWriter(stream=lowercase__       ) as writer:
								writer._build_writer(inferred_schema=lowercase__       )
								assert writer._schema == pa.schema([pa.field('''col_1'''     ,					pa.string()       )]       )













from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tensorflow_text_available, is_torch_available


_UpperCAmelCase			:		Optional[Any]   				=    {
    "configuration_ernie": ["ERNIE_PRETRAINED_CONFIG_ARCHIVE_MAP", "ErnieConfig", "ErnieOnnxConfig"],
}

try:
			if not is_torch_available():
						raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
			pass
else:
			_UpperCAmelCase			:		Any   				=    [
			    "ERNIE_PRETRAINED_MODEL_ARCHIVE_LIST",
			    "ErnieForCausalLM",
			    "ErnieForMaskedLM",
			    "ErnieForMultipleChoice",
			    "ErnieForNextSentencePrediction",
			    "ErnieForPreTraining",
			    "ErnieForQuestionAnswering",
			    "ErnieForSequenceClassification",
			    "ErnieForTokenClassification",
			    "ErnieModel",
			    "ErniePreTrainedModel",
			]

if TYPE_CHECKING:
			from .configuration_ernie import ERNIE_PRETRAINED_CONFIG_ARCHIVE_MAP, ErnieConfig, ErnieOnnxConfig

			try:
						if not is_torch_available():
									raise OptionalDependencyNotAvailable()
			except OptionalDependencyNotAvailable:
						pass
			else:
						from .modeling_ernie import (
						    ERNIE_PRETRAINED_MODEL_ARCHIVE_LIST,
						    ErnieForCausalLM,
						    ErnieForMaskedLM,
						    ErnieForMultipleChoice,
						    ErnieForNextSentencePrediction,
						    ErnieForPreTraining,
						    ErnieForQuestionAnswering,
						    ErnieForSequenceClassification,
						    ErnieForTokenClassification,
						    ErnieModel,
						    ErniePreTrainedModel,
						)

else:
			import sys

			_UpperCAmelCase			:		Dict   				=    _LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)





