






from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


_A						      =				{
    "configuration_distilbert": [
        "DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP",
        "DistilBertConfig",
        "DistilBertOnnxConfig",
    ],
    "tokenization_distilbert": ["DistilBertTokenizer"],
}

try:
   if not is_tokenizers_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   _A						      =				["DistilBertTokenizerFast"]

try:
   if not is_torch_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   _A						      =				[
       "DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST",
       "DistilBertForMaskedLM",
       "DistilBertForMultipleChoice",
       "DistilBertForQuestionAnswering",
       "DistilBertForSequenceClassification",
       "DistilBertForTokenClassification",
       "DistilBertModel",
       "DistilBertPreTrainedModel",
   ]

try:
   if not is_tf_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   _A						      =				[
       "TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST",
       "TFDistilBertForMaskedLM",
       "TFDistilBertForMultipleChoice",
       "TFDistilBertForQuestionAnswering",
       "TFDistilBertForSequenceClassification",
       "TFDistilBertForTokenClassification",
       "TFDistilBertMainLayer",
       "TFDistilBertModel",
       "TFDistilBertPreTrainedModel",
   ]

try:
   if not is_flax_available():
      raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
   pass
else:
   _A						      =				[
       "FlaxDistilBertForMaskedLM",
       "FlaxDistilBertForMultipleChoice",
       "FlaxDistilBertForQuestionAnswering",
       "FlaxDistilBertForSequenceClassification",
       "FlaxDistilBertForTokenClassification",
       "FlaxDistilBertModel",
       "FlaxDistilBertPreTrainedModel",
   ]


if TYPE_CHECKING:
   from .configuration_distilbert import (
       DISTILBERT_PRETRAINED_CONFIG_ARCHIVE_MAP,
       DistilBertConfig,
       DistilBertOnnxConfig,
   )
   from .tokenization_distilbert import DistilBertTokenizer

   try:
      if not is_tokenizers_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .tokenization_distilbert_fast import DistilBertTokenizerFast

   try:
      if not is_torch_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_distilbert import (
          DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
          DistilBertForMaskedLM,
          DistilBertForMultipleChoice,
          DistilBertForQuestionAnswering,
          DistilBertForSequenceClassification,
          DistilBertForTokenClassification,
          DistilBertModel,
          DistilBertPreTrainedModel,
      )

   try:
      if not is_tf_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_tf_distilbert import (
          TF_DISTILBERT_PRETRAINED_MODEL_ARCHIVE_LIST,
          TFDistilBertForMaskedLM,
          TFDistilBertForMultipleChoice,
          TFDistilBertForQuestionAnswering,
          TFDistilBertForSequenceClassification,
          TFDistilBertForTokenClassification,
          TFDistilBertMainLayer,
          TFDistilBertModel,
          TFDistilBertPreTrainedModel,
      )

   try:
      if not is_flax_available():
         raise OptionalDependencyNotAvailable()
   except OptionalDependencyNotAvailable:
      pass
   else:
      from .modeling_flax_distilbert import (
          FlaxDistilBertForMaskedLM,
          FlaxDistilBertForMultipleChoice,
          FlaxDistilBertForQuestionAnswering,
          FlaxDistilBertForSequenceClassification,
          FlaxDistilBertForTokenClassification,
          FlaxDistilBertModel,
          FlaxDistilBertPreTrainedModel,
      )

else:
   import sys

   _A						      =				_LazyModule(__name__, globals()["__file__"], _import_structure, module_spec=__spec__)



import copy
import os
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pyarrow as pa
import pyarrow.parquet as pq
import pytest

from datasets.arrow_writer import ArrowWriter, OptimizedTypedSequence, ParquetWriter, TypedSequence
from datasets.features import ArrayaD, ClassLabel, Features, Image, Value
from datasets.features.features import ArrayaDExtensionType, cast_to_python_objects
from datasets.keyhash import DuplicatedKeysError, InvalidKeyError

from .utils import require_pil







class   lowerCamelCase	(						A_						):





     def 	UpperCAmelCase(self   :					str     )		->		Dict:
         snake_case				  =      pa.array(TypedSequence([1, 2, 3]     )     )
         self.assertEqual(arr.type       ,					pa.intaa()     )





     def 	UpperCAmelCase(self   :					Optional[int]     )		->		Optional[int]:
         with self.assertRaises(_A     ):
             snake_case				  =      pa.array(TypedSequence([1, 2, 3]     )       ,					type=pa.intaa()     )





     def 	UpperCAmelCase(self   :					Dict     )		->		str:
         with self.assertRaises(_A     ):
             snake_case				  =      pa.array(TypedSequence([1, 2, 3]       ,					try_type=Value("bool"     )       ,					type=Value("int64"     )     )     )





     def 	UpperCAmelCase(self   :					List[Any]     )		->		List[str]:
         snake_case				  =      pa.array(TypedSequence([1, 2, 3]       ,					type=Value("int32"     )     )     )
         self.assertEqual(arr.type       ,					pa.intaa()     )





     def 	UpperCAmelCase(self   :					Tuple     )		->		List[Any]:
         with self.assertRaises((TypeError, pa.lib.ArrowInvalid)     ):
             snake_case				  =      pa.array(TypedSequence(["foo", "bar"]       ,					type=Value("int64"     )     )     )





     def 	UpperCAmelCase(self   :					List[Any]     )		->		Dict:
         snake_case				  =      pa.array(TypedSequence([1, 2, 3]       ,					try_type=Value("int32"     )     )     )
         self.assertEqual(arr.type       ,					pa.intaa()     )





     def 	UpperCAmelCase(self   :					Optional[int]     )		->		Optional[int]:
         snake_case				  =      pa.array(TypedSequence(["foo", "bar"]       ,					try_type=Value("int64"     )     )     )
         self.assertEqual(arr.type       ,					pa.string()     )





     def 	UpperCAmelCase(self   :					Any     )		->		Tuple:
         snake_case				  =      pa.array(TypedSequence([[[1, 2, 3]]]       ,					type=ArrayaD((1, 3)       ,					"int64"     )     )     )
         self.assertEqual(arr.type       ,					ArrayaDExtensionType((1, 3)       ,					"int64"     )     )





     def 	UpperCAmelCase(self   :					str     )		->		Union[str, Any]:
         with self.assertRaises((TypeError, pa.lib.ArrowInvalid)     ):
             snake_case				  =      pa.array(TypedSequence(["foo", "bar"]       ,					type=ArrayaD((1, 3)       ,					"int64"     )     )     )





     def 	UpperCAmelCase(self   :					Optional[int]     )		->		Tuple:
         snake_case				  =      pa.array(TypedSequence([[[1, 2, 3]]]       ,					try_type=ArrayaD((1, 3)       ,					"int64"     )     )     )
         self.assertEqual(arr.type       ,					ArrayaDExtensionType((1, 3)       ,					"int64"     )     )





     def 	UpperCAmelCase(self   :					str     )		->		Dict:
         snake_case				  =      pa.array(TypedSequence(["foo", "bar"]       ,					try_type=ArrayaD((1, 3)       ,					"int64"     )     )     )
         self.assertEqual(arr.type       ,					pa.string()     )





     @require_pil
     def 	UpperCAmelCase(self   :					Optional[int]     )		->		Any:
         import PIL.Image

         snake_case				  =      PIL.Image.fromarray(np.arange(1_0       ,					dtype=np.uinta     ).reshape(2       ,					5     )     )
         with patch(
             "datasets.arrow_writer.cast_to_python_objects"       ,					side_effect=_A     ) as mock_cast_to_python_objects:
             snake_case				  =      pa.array(TypedSequence([{"path": None, "bytes": b"image_bytes"}, pil_image]       ,					type=Image()     )     )
             snake_case  ,							snake_case				  =      mock_cast_to_python_objects.call_args_list[-1]
             self.assertIn("optimize_list_casting"       ,					_A     )
             self.assertFalse(kwargs["optimize_list_casting"]     )



def 		lowercase_    (     A__				,	A__	)							->							Tuple:
    """simple docstring"""

    snake_case				  =      pa.BufferReader(A__	) if isinstance(A__				,	pa.Buffer	) else pa.memory_map(A__	)
    snake_case				  =      pa.ipc.open_stream(A__	)
    snake_case				  =      f.read_all()
    assert len(pa_table.to_batches()	) == expected_num_chunks
    assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}
    del pa_table



@pytest.mark.parametrize("writer_batch_size"				,	[None, 1, 10]	)
@pytest.mark.parametrize(
    "fields"				,	[None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}]	)
def 		lowercase_    (     A__				,	A__	)							->							Optional[int]:
    """simple docstring"""

    snake_case				  =      pa.BufferOutputStream()
    snake_case				  =      pa.schema(A__	) if fields else None
    with ArrowWriter(stream=A__				,	schema=A__				,	writer_batch_size=A__	) as writer:
        writer.write({"col_1": "foo", "col_2": 1}	)
        writer.write({"col_1": "bar", "col_2": 2}	)
        snake_case  ,							snake_case				  =      writer.finalize()
    assert num_examples == 2
    assert num_bytes > 0
    if not fields:
        snake_case				  =      {"col_1": pa.string(), "col_2": pa.intaa()}
    assert writer._schema == pa.schema(A__				,	metadata=writer._schema.metadata	)
    _check_output(output.getvalue()				,	expected_num_chunks=num_examples if writer_batch_size == 1 else 1	)



def 		lowercase_    (     )							->							str:
    """simple docstring"""

    snake_case				  =      pa.BufferOutputStream()
    snake_case				  =      Features({"labels": ClassLabel(names=["neg", "pos"]	)}	)
    with ArrowWriter(stream=A__				,	features=A__	) as writer:
        writer.write({"labels": 0}	)
        writer.write({"labels": 1}	)
        snake_case  ,							snake_case				  =      writer.finalize()
    assert num_examples == 2
    assert num_bytes > 0
    assert writer._schema == features.arrow_schema
    assert writer._schema.metadata == features.arrow_schema.metadata
    snake_case				  =      pa.BufferReader(output.getvalue()	)
    snake_case				  =      pa.ipc.open_stream(A__	)
    snake_case				  =      f.read_all()
    snake_case				  =      pa_table.schema
    assert pa_table.num_rows == 2
    assert schema == features.arrow_schema
    assert schema.metadata == features.arrow_schema.metadata
    assert features == Features.from_arrow_schema(A__	)



@pytest.mark.parametrize("writer_batch_size"				,	[None, 1, 10]	)
def 		lowercase_    (     A__	)							->							int:
    """simple docstring"""

    snake_case				  =      pa.BufferOutputStream()
    with ArrowWriter(
        stream=A__				,	writer_batch_size=A__				,	hash_salt="split_name"				,	check_duplicates=A__				,	) as writer:
        with pytest.raises(A__	):
            writer.write({"col_1": "foo", "col_2": 1}				,	key=[1, 2]	)
            snake_case  ,							snake_case				  =      writer.finalize()



@pytest.mark.parametrize("writer_batch_size"				,	[None, 2, 10]	)
def 		lowercase_    (     A__	)							->							str:
    """simple docstring"""

    snake_case				  =      pa.BufferOutputStream()
    with ArrowWriter(
        stream=A__				,	writer_batch_size=A__				,	hash_salt="split_name"				,	check_duplicates=A__				,	) as writer:
        with pytest.raises(A__	):
            writer.write({"col_1": "foo", "col_2": 1}				,	key=10	)
            writer.write({"col_1": "bar", "col_2": 2}				,	key=10	)
            snake_case  ,							snake_case				  =      writer.finalize()



@pytest.mark.parametrize("writer_batch_size"				,	[None, 2, 10]	)
def 		lowercase_    (     A__	)							->							Optional[int]:
    """simple docstring"""

    snake_case				  =      pa.BufferOutputStream()
    with ArrowWriter(
        stream=A__				,	writer_batch_size=A__				,	hash_salt="split_name"				,	check_duplicates=A__				,	) as writer:
        writer.write({"col_1": "foo", "col_2": 1}				,	key=1	)
        writer.write({"col_1": "bar", "col_2": 2}				,	key=2	)
        snake_case  ,							snake_case				  =      writer.finalize()
    assert num_examples == 2
    assert num_bytes > 0
    _check_output(output.getvalue()				,	expected_num_chunks=num_examples if writer_batch_size == 1 else 1	)



@pytest.mark.parametrize("writer_batch_size"				,	[None, 1, 10]	)
@pytest.mark.parametrize(
    "fields"				,	[None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}]	)
def 		lowercase_    (     A__				,	A__	)							->							Dict:
    """simple docstring"""

    snake_case				  =      pa.BufferOutputStream()
    snake_case				  =      pa.schema(A__	) if fields else None
    with ArrowWriter(stream=A__				,	schema=A__				,	writer_batch_size=A__	) as writer:
        writer.write_batch({"col_1": ["foo", "bar"], "col_2": [1, 2]}	)
        writer.write_batch({"col_1": [], "col_2": []}	)
        snake_case  ,							snake_case				  =      writer.finalize()
    assert num_examples == 2
    assert num_bytes > 0
    if not fields:
        snake_case				  =      {"col_1": pa.string(), "col_2": pa.intaa()}
    assert writer._schema == pa.schema(A__				,	metadata=writer._schema.metadata	)
    _check_output(output.getvalue()				,	expected_num_chunks=num_examples if writer_batch_size == 1 else 1	)



@pytest.mark.parametrize("writer_batch_size"				,	[None, 1, 10]	)
@pytest.mark.parametrize(
    "fields"				,	[None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}]	)
def 		lowercase_    (     A__				,	A__	)							->							Union[str, Any]:
    """simple docstring"""

    snake_case				  =      pa.BufferOutputStream()
    snake_case				  =      pa.schema(A__	) if fields else None
    with ArrowWriter(stream=A__				,	schema=A__				,	writer_batch_size=A__	) as writer:
        writer.write_table(pa.Table.from_pydict({"col_1": ["foo", "bar"], "col_2": [1, 2]}	)	)
        snake_case  ,							snake_case				  =      writer.finalize()
    assert num_examples == 2
    assert num_bytes > 0
    if not fields:
        snake_case				  =      {"col_1": pa.string(), "col_2": pa.intaa()}
    assert writer._schema == pa.schema(A__				,	metadata=writer._schema.metadata	)
    _check_output(output.getvalue()				,	expected_num_chunks=num_examples if writer_batch_size == 1 else 1	)



@pytest.mark.parametrize("writer_batch_size"				,	[None, 1, 10]	)
@pytest.mark.parametrize(
    "fields"				,	[None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}]	)
def 		lowercase_    (     A__				,	A__	)							->							List[Any]:
    """simple docstring"""

    snake_case				  =      pa.BufferOutputStream()
    snake_case				  =      pa.schema(A__	) if fields else None
    with ArrowWriter(stream=A__				,	schema=A__				,	writer_batch_size=A__	) as writer:
        writer.write_row(pa.Table.from_pydict({"col_1": ["foo"], "col_2": [1]}	)	)
        writer.write_row(pa.Table.from_pydict({"col_1": ["bar"], "col_2": [2]}	)	)
        snake_case  ,							snake_case				  =      writer.finalize()
    assert num_examples == 2
    assert num_bytes > 0
    if not fields:
        snake_case				  =      {"col_1": pa.string(), "col_2": pa.intaa()}
    assert writer._schema == pa.schema(A__				,	metadata=writer._schema.metadata	)
    _check_output(output.getvalue()				,	expected_num_chunks=num_examples if writer_batch_size == 1 else 1	)



def 		lowercase_    (     )							->							Dict:
    """simple docstring"""

    with tempfile.TemporaryDirectory() as tmp_dir:
        snake_case				  =      {"col_1": pa.string(), "col_2": pa.intaa()}
        snake_case				  =      os.path.join(A__				,	"test.arrow"	)
        with ArrowWriter(path=A__				,	schema=pa.schema(A__	)	) as writer:
            writer.write_batch({"col_1": ["foo", "bar"], "col_2": [1, 2]}	)
            snake_case  ,							snake_case				  =      writer.finalize()
        assert num_examples == 2
        assert num_bytes > 0
        assert writer._schema == pa.schema(A__				,	metadata=writer._schema.metadata	)
        _check_output(A__				,	1	)



def 		lowercase_    (     A__	)							->							int:
    """simple docstring"""

    if pa.types.is_list(A__	):
        return get_base_dtype(arr_type.value_type	)
    else:
        return arr_type



def 		lowercase_    (     A__				,	A__	)							->							Any:
    """simple docstring"""

    if isinstance(lst[0]				,	A__	):
        change_first_primitive_element_in_list(lst[0]				,	A__	)
    else:
        snake_case				  =      value



@pytest.mark.parametrize("optimized_int_type, expected_dtype"				,	[(None, pa.intaa()), (Value("int32"	), pa.intaa())]	)
@pytest.mark.parametrize("sequence"				,	[[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]]	)
def 		lowercase_    (     A__				,	A__				,	A__	)							->							Tuple:
    """simple docstring"""

    snake_case				  =      pa.array(TypedSequence(A__				,	optimized_int_type=A__	)	)
    assert get_base_dtype(arr.type	) == expected_dtype



@pytest.mark.parametrize(
    "col, expected_dtype"				,	[
        ("attention_mask", pa.inta()),
        ("special_tokens_mask", pa.inta()),
        ("token_type_ids", pa.inta()),
        ("input_ids", pa.intaa()),
        ("other", pa.intaa()),
    ]				,	)
@pytest.mark.parametrize("sequence"				,	[[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]]	)
def 		lowercase_    (     A__				,	A__				,	A__	)							->							List[str]:
    """simple docstring"""

    snake_case				  =      pa.array(OptimizedTypedSequence(A__				,	col=A__	)	)
    assert get_base_dtype(arr.type	) == expected_dtype

    # not in range
    if col != "other":
        # avoids errors due to in-place modifications
        snake_case				  =      copy.deepcopy(A__	)
        snake_case				  =      np.iinfo(expected_dtype.to_pandas_dtype()	).max + 1
        change_first_primitive_element_in_list(A__				,	A__	)
        snake_case				  =      pa.array(OptimizedTypedSequence(A__				,	col=A__	)	)
        assert get_base_dtype(arr.type	) == pa.intaa()



@pytest.mark.parametrize("raise_exception"				,	[False, True]	)
def 		lowercase_    (     A__				,	A__	)							->							Optional[Any]:
    """simple docstring"""

    snake_case				  =      str(tmp_path / "dataset-train.arrow"	)
    try:
        with ArrowWriter(path=A__	) as writer:
            if raise_exception:
                raise pa.lib.ArrowInvalid()
            else:
                writer.stream.close()
    except pa.lib.ArrowInvalid:
        pass
    finally:
        assert writer.stream.closed



def 		lowercase_    (     A__	)							->							Optional[int]:
    """simple docstring"""

    snake_case				  =      "mock://dataset-train.arrow"
    with ArrowWriter(path=A__				,	storage_options=mockfs.storage_options	) as writer:
        assert isinstance(writer._fs				,	type(A__	)	)
        assert writer._fs.storage_options == mockfs.storage_options
        writer.write({"col_1": "foo", "col_2": 1}	)
        writer.write({"col_1": "bar", "col_2": 2}	)
        snake_case  ,							snake_case				  =      writer.finalize()
    assert num_examples == 2
    assert num_bytes > 0
    assert mockfs.exists(A__	)



def 		lowercase_    (     )							->							Tuple:
    """simple docstring"""

    snake_case				  =      pa.BufferOutputStream()
    with ParquetWriter(stream=A__	) as writer:
        writer.write({"col_1": "foo", "col_2": 1}	)
        writer.write({"col_1": "bar", "col_2": 2}	)
        snake_case  ,							snake_case				  =      writer.finalize()
    assert num_examples == 2
    assert num_bytes > 0
    snake_case				  =      pa.BufferReader(output.getvalue()	)
    snake_case				  =      pq.read_table(A__	)
    assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}



@require_pil
@pytest.mark.parametrize("embed_local_files"				,	[False, True]	)
def 		lowercase_    (     A__				,	A__	)							->							Dict:
    """simple docstring"""

    import PIL.Image

    snake_case				  =      str(tmp_path / "test_image_rgb.jpg"	)
    PIL.Image.fromarray(np.zeros((5, 5)				,	dtype=np.uinta	)	).save(A__				,	format="png"	)
    snake_case				  =      pa.BufferOutputStream()
    with ParquetWriter(
        stream=A__				,	features=Features({"image": Image()}	)				,	embed_local_files=A__	) as writer:
        writer.write({"image": image_path}	)
        writer.finalize()
    snake_case				  =      pa.BufferReader(output.getvalue()	)
    snake_case				  =      pq.read_table(A__	)
    snake_case				  =      pa_table.to_pydict()
    if embed_local_files:
        assert isinstance(out["image"][0]["path"]				,	A__	)
        with open(A__				,	"rb"	) as f:
            assert out["image"][0]["bytes"] == f.read()
    else:
        assert out["image"][0]["path"] == image_path
        assert out["image"][0]["bytes"] is None





def 		lowercase_    (     )							->							List[Any]:
    """simple docstring"""

    snake_case				  =      pa.schema([pa.field("col_1"				,	pa.string()				,	nullable=A__	)]	)
    snake_case				  =      pa.BufferOutputStream()
    with ArrowWriter(stream=A__	) as writer:
        writer._build_writer(inferred_schema=A__	)
        assert writer._schema == pa.schema([pa.field("col_1"				,	pa.string()	)]	)

