

import copy
import os
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pyarrow as pa
import pyarrow.parquet as pq
import pytest

from datasets.arrow_writer import ArrowWriter, OptimizedTypedSequence, ParquetWriter, TypedSequence
from datasets.features import ArrayaD, ClassLabel, Features, Image, Value
from datasets.features.features import ArrayaDExtensionType, cast_to_python_objects
from datasets.keyhash import DuplicatedKeysError, InvalidKeyError

from .utils import require_pil





class 				__lowerCAmelCase  (      SCREAMING_SNAKE_CASE							):




  def 						A__  (       self      )						->				int:
        '''simple docstring'''






        _lowercase			 =pa.array(TypedSequence([1, 2, 3]      )      )
        self.assertEqual(arr.type							,     pa.intaa()      )




  def 						A__  (       self      )						->				List[Any]:
        '''simple docstring'''






        with self.assertRaises(lowerCAmelCase      ):
              _lowercase			 =pa.array(TypedSequence([1, 2, 3]      )							,     type=pa.intaa()      )




  def 						A__  (       self      )						->				Dict:
        '''simple docstring'''






        with self.assertRaises(lowerCAmelCase      ):
              _lowercase			 =pa.array(TypedSequence([1, 2, 3]							,     try_type=Value('bool'      )							,     type=Value('int64'      )      )      )




  def 						A__  (       self      )						->				Dict:
        '''simple docstring'''






        _lowercase			 =pa.array(TypedSequence([1, 2, 3]							,     type=Value('int32'      )      )      )
        self.assertEqual(arr.type							,     pa.intaa()      )




  def 						A__  (       self      )						->				List[Any]:
        '''simple docstring'''






        with self.assertRaises((TypeError, pa.lib.ArrowInvalid)      ):
              _lowercase			 =pa.array(TypedSequence(['foo', 'bar']							,     type=Value('int64'      )      )      )




  def 						A__  (       self      )						->				List[str]:
        '''simple docstring'''






        _lowercase			 =pa.array(TypedSequence([1, 2, 3]							,     try_type=Value('int32'      )      )      )
        self.assertEqual(arr.type							,     pa.intaa()      )




  def 						A__  (       self      )						->				Tuple:
        '''simple docstring'''






        _lowercase			 =pa.array(TypedSequence(['foo', 'bar']							,     try_type=Value('int64'      )      )      )
        self.assertEqual(arr.type							,     pa.string()      )




  def 						A__  (       self      )						->				Optional[int]:
        '''simple docstring'''






        _lowercase			 =pa.array(TypedSequence([[[1, 2, 3]]]							,     type=ArrayaD((1, 3)							,     'int64'      )      )      )
        self.assertEqual(arr.type							,     ArrayaDExtensionType((1, 3)							,     'int64'      )      )




  def 						A__  (       self      )						->				Any:
        '''simple docstring'''






        with self.assertRaises((TypeError, pa.lib.ArrowInvalid)      ):
              _lowercase			 =pa.array(TypedSequence(['foo', 'bar']							,     type=ArrayaD((1, 3)							,     'int64'      )      )      )




  def 						A__  (       self      )						->				Optional[int]:
        '''simple docstring'''






        _lowercase			 =pa.array(TypedSequence([[[1, 2, 3]]]							,     try_type=ArrayaD((1, 3)							,     'int64'      )      )      )
        self.assertEqual(arr.type							,     ArrayaDExtensionType((1, 3)							,     'int64'      )      )




  def 						A__  (       self      )						->				Union[str, Any]:
        '''simple docstring'''






        _lowercase			 =pa.array(TypedSequence(['foo', 'bar']							,     try_type=ArrayaD((1, 3)							,     'int64'      )      )      )
        self.assertEqual(arr.type							,     pa.string()      )




  @require_pil
  def 						A__  (       self      )						->				List[Any]:
        '''simple docstring'''






        import PIL.Image

        _lowercase			 =PIL.Image.fromarray(np.arange(10							,     dtype=np.uinta      ).reshape(2							,     5      )      )
        with patch(
            'datasets.arrow_writer.cast_to_python_objects'							,     side_effect=lowerCAmelCase      ) as mock_cast_to_python_objects:
              _lowercase			 =pa.array(TypedSequence([{'path': None, 'bytes': B'image_bytes'}, pil_image]							,     type=Image()      )      )
              _lowercase , _lowercase			 =mock_cast_to_python_objects.call_args_list[-1]
              self.assertIn('optimize_list_casting'							,     lowerCAmelCase      )
              self.assertFalse(kwargs['optimize_list_casting']      )
def   a   (    A__			:     List[Any]   ,     A__			:     int  )					->   Any:





      """simple docstring"""



      _lowercase			 =pa.BufferReader(A__  ) if isinstance(A__   ,     pa.Buffer  ) else pa.memory_map(A__  )
      _lowercase			 =pa.ipc.open_stream(A__  )
      _lowercase			 =f.read_all()
      assert len(pa_table.to_batches()  ) == expected_num_chunks
      assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}
      del pa_table
@pytest.mark.parametrize('writer_batch_size'   ,     [None, 1, 10]  )
@pytest.mark.parametrize(
    'fields'   ,     [None, {'col_1': pa.string(), 'col_2': pa.intaa()}, {'col_1': pa.string(), 'col_2': pa.intaa()}]  )
def   a   (    A__			:     List[str]   ,     A__			:     Union[str, Any]  )					->   Any:





      """simple docstring"""



      _lowercase			 =pa.BufferOutputStream()
      _lowercase			 =pa.schema(A__  ) if fields else None
      with ArrowWriter(stream=A__   ,     schema=A__   ,     writer_batch_size=A__  ) as writer:
            writer.write({'col_1': 'foo', 'col_2': 1}  )
            writer.write({'col_1': 'bar', 'col_2': 2}  )
            _lowercase , _lowercase			 =writer.finalize()
      assert num_examples == 2
      assert num_bytes > 0
      if not fields:
            _lowercase			 ={'col_1': pa.string(), 'col_2': pa.intaa()}
      assert writer._schema == pa.schema(A__   ,     metadata=writer._schema.metadata  )
      _check_output(output.getvalue()   ,     expected_num_chunks=num_examples if writer_batch_size == 1 else 1  )
def   a   (    )					->   Any:





      """simple docstring"""



      _lowercase			 =pa.BufferOutputStream()
      _lowercase			 =Features({'labels': ClassLabel(names=['neg', 'pos']  )}  )
      with ArrowWriter(stream=A__   ,     features=A__  ) as writer:
            writer.write({'labels': 0}  )
            writer.write({'labels': 1}  )
            _lowercase , _lowercase			 =writer.finalize()
      assert num_examples == 2
      assert num_bytes > 0
      assert writer._schema == features.arrow_schema
      assert writer._schema.metadata == features.arrow_schema.metadata
      _lowercase			 =pa.BufferReader(output.getvalue()  )
      _lowercase			 =pa.ipc.open_stream(A__  )
      _lowercase			 =f.read_all()
      _lowercase			 =pa_table.schema
      assert pa_table.num_rows == 2
      assert schema == features.arrow_schema
      assert schema.metadata == features.arrow_schema.metadata
      assert features == Features.from_arrow_schema(A__  )
@pytest.mark.parametrize('writer_batch_size'   ,     [None, 1, 10]  )
def   a   (    A__			:     int  )					->   List[Any]:





      """simple docstring"""



      _lowercase			 =pa.BufferOutputStream()
      with ArrowWriter(
          stream=A__   ,     writer_batch_size=A__   ,     hash_salt='split_name'   ,     check_duplicates=A__   ,     ) as writer:
            with pytest.raises(A__  ):
                  writer.write({'col_1': 'foo', 'col_2': 1}   ,     key=[1, 2]  )
                  _lowercase , _lowercase			 =writer.finalize()
@pytest.mark.parametrize('writer_batch_size'   ,     [None, 2, 10]  )
def   a   (    A__			:     int  )					->   List[Any]:





      """simple docstring"""



      _lowercase			 =pa.BufferOutputStream()
      with ArrowWriter(
          stream=A__   ,     writer_batch_size=A__   ,     hash_salt='split_name'   ,     check_duplicates=A__   ,     ) as writer:
            with pytest.raises(A__  ):
                  writer.write({'col_1': 'foo', 'col_2': 1}   ,     key=10  )
                  writer.write({'col_1': 'bar', 'col_2': 2}   ,     key=10  )
                  _lowercase , _lowercase			 =writer.finalize()
@pytest.mark.parametrize('writer_batch_size'   ,     [None, 2, 10]  )
def   a   (    A__			:     Optional[Any]  )					->   Optional[int]:





      """simple docstring"""



      _lowercase			 =pa.BufferOutputStream()
      with ArrowWriter(
          stream=A__   ,     writer_batch_size=A__   ,     hash_salt='split_name'   ,     check_duplicates=A__   ,     ) as writer:
            writer.write({'col_1': 'foo', 'col_2': 1}   ,     key=1  )
            writer.write({'col_1': 'bar', 'col_2': 2}   ,     key=2  )
            _lowercase , _lowercase			 =writer.finalize()
      assert num_examples == 2
      assert num_bytes > 0
      _check_output(output.getvalue()   ,     expected_num_chunks=num_examples if writer_batch_size == 1 else 1  )
@pytest.mark.parametrize('writer_batch_size'   ,     [None, 1, 10]  )
@pytest.mark.parametrize(
    'fields'   ,     [None, {'col_1': pa.string(), 'col_2': pa.intaa()}, {'col_1': pa.string(), 'col_2': pa.intaa()}]  )
def   a   (    A__			:     Optional[Any]   ,     A__			:     List[str]  )					->   Any:





      """simple docstring"""



      _lowercase			 =pa.BufferOutputStream()
      _lowercase			 =pa.schema(A__  ) if fields else None
      with ArrowWriter(stream=A__   ,     schema=A__   ,     writer_batch_size=A__  ) as writer:
            writer.write_batch({'col_1': ['foo', 'bar'], 'col_2': [1, 2]}  )
            writer.write_batch({'col_1': [], 'col_2': []}  )
            _lowercase , _lowercase			 =writer.finalize()
      assert num_examples == 2
      assert num_bytes > 0
      if not fields:
            _lowercase			 ={'col_1': pa.string(), 'col_2': pa.intaa()}
      assert writer._schema == pa.schema(A__   ,     metadata=writer._schema.metadata  )
      _check_output(output.getvalue()   ,     expected_num_chunks=num_examples if writer_batch_size == 1 else 1  )
@pytest.mark.parametrize('writer_batch_size'   ,     [None, 1, 10]  )
@pytest.mark.parametrize(
    'fields'   ,     [None, {'col_1': pa.string(), 'col_2': pa.intaa()}, {'col_1': pa.string(), 'col_2': pa.intaa()}]  )
def   a   (    A__			:     Dict   ,     A__			:     List[str]  )					->   Any:





      """simple docstring"""



      _lowercase			 =pa.BufferOutputStream()
      _lowercase			 =pa.schema(A__  ) if fields else None
      with ArrowWriter(stream=A__   ,     schema=A__   ,     writer_batch_size=A__  ) as writer:
            writer.write_table(pa.Table.from_pydict({'col_1': ['foo', 'bar'], 'col_2': [1, 2]}  )  )
            _lowercase , _lowercase			 =writer.finalize()
      assert num_examples == 2
      assert num_bytes > 0
      if not fields:
            _lowercase			 ={'col_1': pa.string(), 'col_2': pa.intaa()}
      assert writer._schema == pa.schema(A__   ,     metadata=writer._schema.metadata  )
      _check_output(output.getvalue()   ,     expected_num_chunks=num_examples if writer_batch_size == 1 else 1  )
@pytest.mark.parametrize('writer_batch_size'   ,     [None, 1, 10]  )
@pytest.mark.parametrize(
    'fields'   ,     [None, {'col_1': pa.string(), 'col_2': pa.intaa()}, {'col_1': pa.string(), 'col_2': pa.intaa()}]  )
def   a   (    A__			:     int   ,     A__			:     Union[str, Any]  )					->   List[str]:





      """simple docstring"""



      _lowercase			 =pa.BufferOutputStream()
      _lowercase			 =pa.schema(A__  ) if fields else None
      with ArrowWriter(stream=A__   ,     schema=A__   ,     writer_batch_size=A__  ) as writer:
            writer.write_row(pa.Table.from_pydict({'col_1': ['foo'], 'col_2': [1]}  )  )
            writer.write_row(pa.Table.from_pydict({'col_1': ['bar'], 'col_2': [2]}  )  )
            _lowercase , _lowercase			 =writer.finalize()
      assert num_examples == 2
      assert num_bytes > 0
      if not fields:
            _lowercase			 ={'col_1': pa.string(), 'col_2': pa.intaa()}
      assert writer._schema == pa.schema(A__   ,     metadata=writer._schema.metadata  )
      _check_output(output.getvalue()   ,     expected_num_chunks=num_examples if writer_batch_size == 1 else 1  )
def   a   (    )					->   Optional[int]:





      """simple docstring"""



      with tempfile.TemporaryDirectory() as tmp_dir:
            _lowercase			 ={'col_1': pa.string(), 'col_2': pa.intaa()}
            _lowercase			 =os.path.join(A__   ,     'test.arrow'  )
            with ArrowWriter(path=A__   ,     schema=pa.schema(A__  )  ) as writer:
                  writer.write_batch({'col_1': ['foo', 'bar'], 'col_2': [1, 2]}  )
                  _lowercase , _lowercase			 =writer.finalize()
            assert num_examples == 2
            assert num_bytes > 0
            assert writer._schema == pa.schema(A__   ,     metadata=writer._schema.metadata  )
            _check_output(A__   ,     1  )
def   a   (    A__			:     Optional[int]  )					->   int:





      """simple docstring"""



      if pa.types.is_list(A__  ):
            return get_base_dtype(arr_type.value_type  )
      else:
            return arr_type
def   a   (    A__			:     Any   ,     A__			:     Optional[int]  )					->   Union[str, Any]:





      """simple docstring"""



      if isinstance(lst[0]   ,     A__  ):
            change_first_primitive_element_in_list(lst[0]   ,     A__  )
      else:
            _lowercase			 =value
@pytest.mark.parametrize('optimized_int_type, expected_dtype'   ,     [(None, pa.intaa()), (Value('int32'  ), pa.intaa())]  )
@pytest.mark.parametrize('sequence'   ,     [[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]]  )
def   a   (    A__			:     Optional[int]   ,     A__			:     Union[str, Any]   ,     A__			:     Optional[int]  )					->   Optional[Any]:





      """simple docstring"""



      _lowercase			 =pa.array(TypedSequence(A__   ,     optimized_int_type=A__  )  )
      assert get_base_dtype(arr.type  ) == expected_dtype
@pytest.mark.parametrize(
    'col, expected_dtype'   ,     [
        ('attention_mask', pa.inta()),
        ('special_tokens_mask', pa.inta()),
        ('token_type_ids', pa.inta()),
        ('input_ids', pa.intaa()),
        ('other', pa.intaa()),
    ]   ,     )
@pytest.mark.parametrize('sequence'   ,     [[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]]  )
def   a   (    A__			:     List[str]   ,     A__			:     Any   ,     A__			:     Any  )					->   Union[str, Any]:





      """simple docstring"""



      _lowercase			 =pa.array(OptimizedTypedSequence(A__   ,     col=A__  )  )
      assert get_base_dtype(arr.type  ) == expected_dtype

      # not in range
      if col != "other":
            # avoids errors due to in-place modifications
            _lowercase			 =copy.deepcopy(A__  )
            _lowercase			 =np.iinfo(expected_dtype.to_pandas_dtype()  ).max + 1
            change_first_primitive_element_in_list(A__   ,     A__  )
            _lowercase			 =pa.array(OptimizedTypedSequence(A__   ,     col=A__  )  )
            assert get_base_dtype(arr.type  ) == pa.intaa()
@pytest.mark.parametrize('raise_exception'   ,     [False, True]  )
def   a   (    A__			:     List[str]   ,     A__			:     int  )					->   Any:





      """simple docstring"""



      _lowercase			 =str(tmp_path / 'dataset-train.arrow'  )
      try:
            with ArrowWriter(path=A__  ) as writer:
                  if raise_exception:
                        raise pa.lib.ArrowInvalid()
                  else:
                        writer.stream.close()
      except pa.lib.ArrowInvalid:
            pass
      finally:
            assert writer.stream.closed
def   a   (    A__			:     str  )					->   str:





      """simple docstring"""



      _lowercase			 ='mock://dataset-train.arrow'
      with ArrowWriter(path=A__   ,     storage_options=mockfs.storage_options  ) as writer:
            assert isinstance(writer._fs   ,     type(A__  )  )
            assert writer._fs.storage_options == mockfs.storage_options
            writer.write({'col_1': 'foo', 'col_2': 1}  )
            writer.write({'col_1': 'bar', 'col_2': 2}  )
            _lowercase , _lowercase			 =writer.finalize()
      assert num_examples == 2
      assert num_bytes > 0
      assert mockfs.exists(A__  )
def   a   (    )					->   int:





      """simple docstring"""



      _lowercase			 =pa.BufferOutputStream()
      with ParquetWriter(stream=A__  ) as writer:
            writer.write({'col_1': 'foo', 'col_2': 1}  )
            writer.write({'col_1': 'bar', 'col_2': 2}  )
            _lowercase , _lowercase			 =writer.finalize()
      assert num_examples == 2
      assert num_bytes > 0
      _lowercase			 =pa.BufferReader(output.getvalue()  )
      _lowercase			 =pq.read_table(A__  )
      assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}
@require_pil
@pytest.mark.parametrize('embed_local_files'   ,     [False, True]  )
def   a   (    A__			:     str   ,     A__			:     Tuple  )					->   Union[str, Any]:





      """simple docstring"""



      import PIL.Image

      _lowercase			 =str(tmp_path / 'test_image_rgb.jpg'  )
      PIL.Image.fromarray(np.zeros((5, 5)   ,     dtype=np.uinta  )  ).save(A__   ,     format='png'  )
      _lowercase			 =pa.BufferOutputStream()
      with ParquetWriter(
          stream=A__   ,     features=Features({'image': Image()}  )   ,     embed_local_files=A__  ) as writer:
            writer.write({'image': image_path}  )
            writer.finalize()
      _lowercase			 =pa.BufferReader(output.getvalue()  )
      _lowercase			 =pq.read_table(A__  )
      _lowercase			 =pa_table.to_pydict()
      if embed_local_files:
            assert isinstance(out['image'][0]['path']   ,     A__  )
            with open(A__   ,     'rb'  ) as f:
                  assert out["image"][0]["bytes"] == f.read()
      else:
            assert out["image"][0]["path"] == image_path
            assert out["image"][0]["bytes"] is None







def   a   (    )					->   Tuple:





      """simple docstring"""



      _lowercase			 =pa.schema([pa.field('col_1'   ,     pa.string()   ,     nullable=A__  )]  )
      _lowercase			 =pa.BufferOutputStream()
      with ArrowWriter(stream=A__  ) as writer:
            writer._build_writer(inferred_schema=A__  )
            assert writer._schema == pa.schema([pa.field('col_1'   ,     pa.string()  )]  )





import math

import tensorflow as tf
from packaging import version
def   a   (    A__			:     Tuple  )					->   List[Any]:





      """simple docstring"""



      _lowercase			 =tf.convert_to_tensor(A__  )
      _lowercase			 =0.5 * (1.0 + tf.math.erf(x / tf.cast(tf.sqrt(2.0  )   ,     x.dtype  )  ))

      return x * cdf
def   a   (    A__			:     Any  )					->   int:





      """simple docstring"""



      _lowercase			 =tf.convert_to_tensor(A__  )
      _lowercase			 =tf.cast(math.pi   ,     x.dtype  )
      _lowercase			 =tf.cast(0.044715   ,     x.dtype  )
      _lowercase			 =0.5 * (1.0 + tf.tanh(tf.sqrt(2.0 / pi  ) * (x + coeff * tf.pow(A__   ,     3  ))  ))

      return x * cdf
def   a   (    A__			:     Optional[int]  )					->   int:





      """simple docstring"""



      _lowercase			 =tf.convert_to_tensor(A__  )

      return x * tf.tanh(tf.math.softplus(A__  )  )
def   a   (    A__			:     Tuple  )					->   List[str]:





      """simple docstring"""



      _lowercase			 =tf.convert_to_tensor(A__  )
      _lowercase			 =tf.cast(0.044715   ,     x.dtype  )
      _lowercase			 =tf.cast(0.7978845608   ,     x.dtype  )

      return 0.5 * x * (1.0 + tf.tanh(x * coeffa * (1.0 + coeffa * x * x)  ))
def   a   (    A__			:     int  )					->   Any:





      """simple docstring"""



      _lowercase			 =tf.convert_to_tensor(A__  )
      _lowercase			 =tf.cast(1.702   ,     x.dtype  )
      return x * tf.math.sigmoid(coeff * x  )
def   a   (    A__			:     int  )					->   Dict:





      """simple docstring"""



      return tf.clip_by_value(_gelu(A__  )   ,     -10   ,     10  )
def   a   (    A__			:     Optional[int]   ,     A__			:     int=-1  )					->   Optional[Any]:





      """simple docstring"""



      _lowercase , _lowercase			 =tf.split(A__   ,     2   ,     axis=A__  )
      return a * tf.math.sigmoid(A__  )


if version.parse(tf.version.VERSION) >= version.parse('2.4'):
  def   a   (    A__			:     Any  )					->   Dict:





        """simple docstring"""



        return tf.keras.activations.gelu(A__   ,     approximate=A__  )

  lowercase_             =     tf.keras.activations.gelu







  lowercase_             =     approximate_gelu_wrap
else:
  lowercase_             =     _gelu
  lowercase_             =     _gelu_new


lowercase_             =     {
    'gelu': gelu,
    'gelu_10': gelu_aa,
    'gelu_fast': gelu_fast,
    'gelu_new': gelu_new,
    'glu': glu,
    'mish': mish,
    'quick_gelu': quick_gelu,
    'relu': tf.keras.activations.relu,
    'sigmoid': tf.keras.activations.sigmoid,
    'silu': tf.keras.activations.swish,
    'swish': tf.keras.activations.swish,
    'tanh': tf.keras.activations.tanh,
}







def   a   (    A__			:     Optional[int]  )					->   str:





      """simple docstring"""



      if activation_string in ACTaFN:
            return ACTaFN[activation_string]
      else:
            raise KeyError(F'''function {activation_string} not found in ACT2FN mapping {list(ACTaFN.keys()  )}'''  )


