import copy
import os
import tempfile
from unittest import TestCase
from unittest.mock import patch

import numpy as np
import pyarrow as pa
import pyarrow.parquet as pq
import pytest

from datasets.arrow_writer import ArrowWriter, OptimizedTypedSequence, ParquetWriter, TypedSequence
from datasets.features import ArrayaD, ClassLabel, Features, Image, Value
from datasets.features.features import ArrayaDExtensionType, cast_to_python_objects
from datasets.keyhash import DuplicatedKeysError, InvalidKeyError

from .utils import require_pil



class        lowercase_   (					__snake_case		):



							def 	UpperCamelCase	(						self    ):
													_snake_case						:					Any    =   pa.array(TypedSequence([1, 2, 3]    )    )
													self.assertEqual(arr.type				,    pa.intaa()    )



							def 	UpperCamelCase	(						self    ):
													with self.assertRaises(lowercase_    ):
																			_snake_case						:					List[Any]    =   pa.array(TypedSequence([1, 2, 3]    )				,    type=pa.intaa()    )



							def 	UpperCamelCase	(						self    ):
													with self.assertRaises(lowercase_    ):
																			_snake_case						:					Optional[Any]    =   pa.array(TypedSequence([1, 2, 3]				,    try_type=Value("bool"    )				,    type=Value("int64"    )    )    )



							def 	UpperCamelCase	(						self    ):
													_snake_case						:					Union[str, Any]    =   pa.array(TypedSequence([1, 2, 3]				,    type=Value("int32"    )    )    )
													self.assertEqual(arr.type				,    pa.intaa()    )



							def 	UpperCamelCase	(						self    ):
													with self.assertRaises((TypeError, pa.lib.ArrowInvalid)    ):
																			_snake_case						:					int    =   pa.array(TypedSequence(["foo", "bar"]				,    type=Value("int64"    )    )    )



							def 	UpperCamelCase	(						self    ):
													_snake_case						:					Tuple    =   pa.array(TypedSequence([1, 2, 3]				,    try_type=Value("int32"    )    )    )
													self.assertEqual(arr.type				,    pa.intaa()    )



							def 	UpperCamelCase	(						self    ):
													_snake_case						:					str    =   pa.array(TypedSequence(["foo", "bar"]				,    try_type=Value("int64"    )    )    )
													self.assertEqual(arr.type				,    pa.string()    )



							def 	UpperCamelCase	(						self    ):
													_snake_case						:					int    =   pa.array(TypedSequence([[[1, 2, 3]]]				,    type=ArrayaD((1, 3)				,    "int64"    )    )    )
													self.assertEqual(arr.type				,    ArrayaDExtensionType((1, 3)				,    "int64"    )    )



							def 	UpperCamelCase	(						self    ):
													with self.assertRaises((TypeError, pa.lib.ArrowInvalid)    ):
																			_snake_case						:					str    =   pa.array(TypedSequence(["foo", "bar"]				,    type=ArrayaD((1, 3)				,    "int64"    )    )    )



							def 	UpperCamelCase	(						self    ):
													_snake_case						:					Dict    =   pa.array(TypedSequence([[[1, 2, 3]]]				,    try_type=ArrayaD((1, 3)				,    "int64"    )    )    )
													self.assertEqual(arr.type				,    ArrayaDExtensionType((1, 3)				,    "int64"    )    )



							def 	UpperCamelCase	(						self    ):
													_snake_case						:					List[Any]    =   pa.array(TypedSequence(["foo", "bar"]				,    try_type=ArrayaD((1, 3)				,    "int64"    )    )    )
													self.assertEqual(arr.type				,    pa.string()    )



							@require_pil
							def 	UpperCamelCase	(						self    ):
													import PIL.Image

													_snake_case						:					List[str]    =   PIL.Image.fromarray(np.arange(10				,    dtype=np.uinta    ).reshape(2				,    5    )    )
													with patch(
													    "datasets.arrow_writer.cast_to_python_objects"				,    side_effect=lowercase_    ) as mock_cast_to_python_objects:
																			_snake_case						:					Optional[Any]    =   pa.array(TypedSequence([{"path": None, "bytes": B"image_bytes"}, pil_image]				,    type=Image()    )    )
																			_snake_case    ,_snake_case						:					Tuple    =   mock_cast_to_python_objects.call_args_list[-1]
																			self.assertIn("optimize_list_casting"				,    lowercase_    )
																			self.assertFalse(kwargs["optimize_list_casting"]    )




def  snake_case						(__lowercase ,      __lowercase						)	->	Optional[int]:

						'''simple docstring'''



						_snake_case						:					Union[str, Any]    =   pa.BufferReader(__lowercase						) if isinstance(__lowercase ,      pa.Buffer						) else pa.memory_map(__lowercase						)
						_snake_case						:					Union[str, Any]    =   pa.ipc.open_stream(__lowercase						)
						_snake_case						:					pa.Table    =   f.read_all()
						assert len(pa_table.to_batches()						) == expected_num_chunks
						assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}
						del pa_table
@pytest.mark.parametrize("writer_batch_size" ,      [None, 1, 10]						)
@pytest.mark.parametrize(
    "fields" ,      [None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}]						)
def  snake_case						(__lowercase ,      __lowercase						)	->	List[Any]:

						'''simple docstring'''



						_snake_case						:					Tuple    =   pa.BufferOutputStream()
						_snake_case						:					List[str]    =   pa.schema(__lowercase						) if fields else None
						with ArrowWriter(stream=__lowercase ,      schema=__lowercase ,      writer_batch_size=__lowercase						) as writer:
												writer.write({"col_1": "foo", "col_2": 1}						)
												writer.write({"col_1": "bar", "col_2": 2}						)
												_snake_case    ,_snake_case						:					List[str]    =   writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						if not fields:
												_snake_case						:					str    =   {"col_1": pa.string(), "col_2": pa.intaa()}
						assert writer._schema == pa.schema(__lowercase ,      metadata=writer._schema.metadata						)
						_check_output(output.getvalue() ,      expected_num_chunks=num_examples if writer_batch_size == 1 else 1						)
def  snake_case						()	->	int:

						'''simple docstring'''



						_snake_case						:					int    =   pa.BufferOutputStream()
						_snake_case						:					List[str]    =   Features({"labels": ClassLabel(names=["neg", "pos"]						)}						)
						with ArrowWriter(stream=__lowercase ,      features=__lowercase						) as writer:
												writer.write({"labels": 0}						)
												writer.write({"labels": 1}						)
												_snake_case    ,_snake_case						:					Optional[int]    =   writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						assert writer._schema == features.arrow_schema
						assert writer._schema.metadata == features.arrow_schema.metadata
						_snake_case						:					Optional[int]    =   pa.BufferReader(output.getvalue()						)
						_snake_case						:					Dict    =   pa.ipc.open_stream(__lowercase						)
						_snake_case						:					pa.Table    =   f.read_all()
						_snake_case						:					Dict    =   pa_table.schema
						assert pa_table.num_rows == 2
						assert schema == features.arrow_schema
						assert schema.metadata == features.arrow_schema.metadata
						assert features == Features.from_arrow_schema(__lowercase						)
@pytest.mark.parametrize("writer_batch_size" ,      [None, 1, 10]						)
def  snake_case						(__lowercase						)	->	Union[str, Any]:

						'''simple docstring'''



						_snake_case						:					str    =   pa.BufferOutputStream()
						with ArrowWriter(
						    stream=__lowercase ,      writer_batch_size=__lowercase ,      hash_salt="split_name" ,      check_duplicates=__lowercase ,      ) as writer:
												with pytest.raises(__lowercase						):
																		writer.write({"col_1": "foo", "col_2": 1} ,      key=[1, 2]						)
																		_snake_case    ,_snake_case						:					Dict    =   writer.finalize()
@pytest.mark.parametrize("writer_batch_size" ,      [None, 2, 10]						)
def  snake_case						(__lowercase						)	->	Dict:

						'''simple docstring'''



						_snake_case						:					List[Any]    =   pa.BufferOutputStream()
						with ArrowWriter(
						    stream=__lowercase ,      writer_batch_size=__lowercase ,      hash_salt="split_name" ,      check_duplicates=__lowercase ,      ) as writer:
												with pytest.raises(__lowercase						):
																		writer.write({"col_1": "foo", "col_2": 1} ,      key=10						)
																		writer.write({"col_1": "bar", "col_2": 2} ,      key=10						)
																		_snake_case    ,_snake_case						:					Union[str, Any]    =   writer.finalize()
@pytest.mark.parametrize("writer_batch_size" ,      [None, 2, 10]						)
def  snake_case						(__lowercase						)	->	Dict:

						'''simple docstring'''



						_snake_case						:					List[str]    =   pa.BufferOutputStream()
						with ArrowWriter(
						    stream=__lowercase ,      writer_batch_size=__lowercase ,      hash_salt="split_name" ,      check_duplicates=__lowercase ,      ) as writer:
												writer.write({"col_1": "foo", "col_2": 1} ,      key=1						)
												writer.write({"col_1": "bar", "col_2": 2} ,      key=2						)
												_snake_case    ,_snake_case						:					int    =   writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						_check_output(output.getvalue() ,      expected_num_chunks=num_examples if writer_batch_size == 1 else 1						)
@pytest.mark.parametrize("writer_batch_size" ,      [None, 1, 10]						)
@pytest.mark.parametrize(
    "fields" ,      [None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}]						)
def  snake_case						(__lowercase ,      __lowercase						)	->	int:

						'''simple docstring'''



						_snake_case						:					Dict    =   pa.BufferOutputStream()
						_snake_case						:					Optional[int]    =   pa.schema(__lowercase						) if fields else None
						with ArrowWriter(stream=__lowercase ,      schema=__lowercase ,      writer_batch_size=__lowercase						) as writer:
												writer.write_batch({"col_1": ["foo", "bar"], "col_2": [1, 2]}						)
												writer.write_batch({"col_1": [], "col_2": []}						)
												_snake_case    ,_snake_case						:					List[str]    =   writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						if not fields:
												_snake_case						:					Tuple    =   {"col_1": pa.string(), "col_2": pa.intaa()}
						assert writer._schema == pa.schema(__lowercase ,      metadata=writer._schema.metadata						)
						_check_output(output.getvalue() ,      expected_num_chunks=num_examples if writer_batch_size == 1 else 1						)
@pytest.mark.parametrize("writer_batch_size" ,      [None, 1, 10]						)
@pytest.mark.parametrize(
    "fields" ,      [None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}]						)
def  snake_case						(__lowercase ,      __lowercase						)	->	Tuple:

						'''simple docstring'''



						_snake_case						:					Optional[Any]    =   pa.BufferOutputStream()
						_snake_case						:					List[str]    =   pa.schema(__lowercase						) if fields else None
						with ArrowWriter(stream=__lowercase ,      schema=__lowercase ,      writer_batch_size=__lowercase						) as writer:
												writer.write_table(pa.Table.from_pydict({"col_1": ["foo", "bar"], "col_2": [1, 2]}						)						)
												_snake_case    ,_snake_case						:					str    =   writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						if not fields:
												_snake_case						:					List[Any]    =   {"col_1": pa.string(), "col_2": pa.intaa()}
						assert writer._schema == pa.schema(__lowercase ,      metadata=writer._schema.metadata						)
						_check_output(output.getvalue() ,      expected_num_chunks=num_examples if writer_batch_size == 1 else 1						)
@pytest.mark.parametrize("writer_batch_size" ,      [None, 1, 10]						)
@pytest.mark.parametrize(
    "fields" ,      [None, {"col_1": pa.string(), "col_2": pa.intaa()}, {"col_1": pa.string(), "col_2": pa.intaa()}]						)
def  snake_case						(__lowercase ,      __lowercase						)	->	List[str]:

						'''simple docstring'''



						_snake_case						:					Union[str, Any]    =   pa.BufferOutputStream()
						_snake_case						:					Optional[Any]    =   pa.schema(__lowercase						) if fields else None
						with ArrowWriter(stream=__lowercase ,      schema=__lowercase ,      writer_batch_size=__lowercase						) as writer:
												writer.write_row(pa.Table.from_pydict({"col_1": ["foo"], "col_2": [1]}						)						)
												writer.write_row(pa.Table.from_pydict({"col_1": ["bar"], "col_2": [2]}						)						)
												_snake_case    ,_snake_case						:					str    =   writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						if not fields:
												_snake_case						:					Tuple    =   {"col_1": pa.string(), "col_2": pa.intaa()}
						assert writer._schema == pa.schema(__lowercase ,      metadata=writer._schema.metadata						)
						_check_output(output.getvalue() ,      expected_num_chunks=num_examples if writer_batch_size == 1 else 1						)
def  snake_case						()	->	Union[str, Any]:

						'''simple docstring'''



						with tempfile.TemporaryDirectory() as tmp_dir:
												_snake_case						:					int    =   {"col_1": pa.string(), "col_2": pa.intaa()}
												_snake_case						:					Any    =   os.path.join(__lowercase ,      "test.arrow"						)
												with ArrowWriter(path=__lowercase ,      schema=pa.schema(__lowercase						)						) as writer:
																		writer.write_batch({"col_1": ["foo", "bar"], "col_2": [1, 2]}						)
																		_snake_case    ,_snake_case						:					List[str]    =   writer.finalize()
												assert num_examples == 2
												assert num_bytes > 0
												assert writer._schema == pa.schema(__lowercase ,      metadata=writer._schema.metadata						)
												_check_output(__lowercase ,      1						)
def  snake_case						(__lowercase						)	->	List[Any]:

						'''simple docstring'''



						if pa.types.is_list(__lowercase						):
												return get_base_dtype(arr_type.value_type						)
						else:
												return arr_type
def  snake_case						(__lowercase ,      __lowercase						)	->	Optional[Any]:

						'''simple docstring'''



						if isinstance(lst[0] ,      __lowercase						):
												change_first_primitive_element_in_list(lst[0] ,      __lowercase						)
						else:
												_snake_case						:					List[str]    =   value
@pytest.mark.parametrize("optimized_int_type, expected_dtype" ,      [(None, pa.intaa()), (Value("int32"						), pa.intaa())]						)
@pytest.mark.parametrize("sequence" ,      [[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]]						)
def  snake_case						(__lowercase ,      __lowercase ,      __lowercase						)	->	Tuple:

						'''simple docstring'''



						_snake_case						:					Any    =   pa.array(TypedSequence(__lowercase ,      optimized_int_type=__lowercase						)						)
						assert get_base_dtype(arr.type						) == expected_dtype
@pytest.mark.parametrize(
    "col, expected_dtype" ,      [
        ("attention_mask", pa.inta()),
        ("special_tokens_mask", pa.inta()),
        ("token_type_ids", pa.inta()),
        ("input_ids", pa.intaa()),
        ("other", pa.intaa()),
    ] ,      )
@pytest.mark.parametrize("sequence" ,      [[1, 2, 3], [[1, 2, 3]], [[[1, 2, 3]]]]						)
def  snake_case						(__lowercase ,      __lowercase ,      __lowercase						)	->	Dict:

						'''simple docstring'''



						_snake_case						:					List[Any]    =   pa.array(OptimizedTypedSequence(__lowercase ,      col=__lowercase						)						)
						assert get_base_dtype(arr.type						) == expected_dtype

						# not in range
						if col != "other":
												# avoids errors due to in-place modifications
												_snake_case						:					Optional[int]    =   copy.deepcopy(__lowercase						)
												_snake_case						:					Any    =   np.iinfo(expected_dtype.to_pandas_dtype()						).max + 1
												change_first_primitive_element_in_list(__lowercase ,      __lowercase						)
												_snake_case						:					int    =   pa.array(OptimizedTypedSequence(__lowercase ,      col=__lowercase						)						)
												assert get_base_dtype(arr.type						) == pa.intaa()
@pytest.mark.parametrize("raise_exception" ,      [False, True]						)
def  snake_case						(__lowercase ,      __lowercase						)	->	List[Any]:

						'''simple docstring'''



						_snake_case						:					Union[str, Any]    =   str(tmp_path / "dataset-train.arrow"						)
						try:
												with ArrowWriter(path=__lowercase						) as writer:
																		if raise_exception:
																								raise pa.lib.ArrowInvalid()
																		else:
																								writer.stream.close()
						except pa.lib.ArrowInvalid:
												pass
						finally:
												assert writer.stream.closed
def  snake_case						(__lowercase						)	->	Tuple:

						'''simple docstring'''



						_snake_case						:					Optional[Any]    =   "mock://dataset-train.arrow"
						with ArrowWriter(path=__lowercase ,      storage_options=mockfs.storage_options						) as writer:
												assert isinstance(writer._fs ,      type(__lowercase						)						)
												assert writer._fs.storage_options == mockfs.storage_options
												writer.write({"col_1": "foo", "col_2": 1}						)
												writer.write({"col_1": "bar", "col_2": 2}						)
												_snake_case    ,_snake_case						:					List[Any]    =   writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						assert mockfs.exists(__lowercase						)
def  snake_case						()	->	Tuple:

						'''simple docstring'''



						_snake_case						:					List[Any]    =   pa.BufferOutputStream()
						with ParquetWriter(stream=__lowercase						) as writer:
												writer.write({"col_1": "foo", "col_2": 1}						)
												writer.write({"col_1": "bar", "col_2": 2}						)
												_snake_case    ,_snake_case						:					Union[str, Any]    =   writer.finalize()
						assert num_examples == 2
						assert num_bytes > 0
						_snake_case						:					Union[str, Any]    =   pa.BufferReader(output.getvalue()						)
						_snake_case						:					pa.Table    =   pq.read_table(__lowercase						)
						assert pa_table.to_pydict() == {"col_1": ["foo", "bar"], "col_2": [1, 2]}
@require_pil
@pytest.mark.parametrize("embed_local_files" ,      [False, True]						)
def  snake_case						(__lowercase ,      __lowercase						)	->	Union[str, Any]:

						'''simple docstring'''



						import PIL.Image

						_snake_case						:					List[Any]    =   str(tmp_path / "test_image_rgb.jpg"						)
						PIL.Image.fromarray(np.zeros((5, 5) ,      dtype=np.uinta						)						).save(__lowercase ,      format="png"						)
						_snake_case						:					Optional[Any]    =   pa.BufferOutputStream()
						with ParquetWriter(
						    stream=__lowercase ,      features=Features({"image": Image()}						) ,      embed_local_files=__lowercase						) as writer:
												writer.write({"image": image_path}						)
												writer.finalize()
						_snake_case						:					Tuple    =   pa.BufferReader(output.getvalue()						)
						_snake_case						:					pa.Table    =   pq.read_table(__lowercase						)
						_snake_case						:					Optional[int]    =   pa_table.to_pydict()
						if embed_local_files:
												assert isinstance(out["image"][0]["path"] ,      __lowercase						)
												with open(__lowercase ,      "rb"						) as f:
																		assert out["image"][0]["bytes"] == f.read()
						else:
												assert out["image"][0]["path"] == image_path
												assert out["image"][0]["bytes"] is None

def  snake_case						()	->	Union[str, Any]:

						'''simple docstring'''



						_snake_case						:					Dict    =   pa.schema([pa.field("col_1" ,      pa.string() ,      nullable=__lowercase						)]						)
						_snake_case						:					List[Any]    =   pa.BufferOutputStream()
						with ArrowWriter(stream=__lowercase						) as writer:
												writer._build_writer(inferred_schema=__lowercase						)
												assert writer._schema == pa.schema([pa.field("col_1" ,      pa.string()						)]						)
from __future__ import annotations

import string
from itertools import cycle, product
from pathlib import Path

__SCREAMING_SNAKE_CASE						:   str         =			(
    string.ascii_letters + string.digits + string.punctuation + string.whitespace
)
__SCREAMING_SNAKE_CASE						:   list[int]         =			[ord(letter) for letter in string.ascii_lowercase]
__SCREAMING_SNAKE_CASE						:   set[int]         =			{ord(char) for char in VALID_CHARS}

__SCREAMING_SNAKE_CASE						:   list[str]         =			["the", "be", "to", "of", "and", "in", "that", "have"]
def  snake_case						(__lowercase ,      __lowercase						)	->	str | None:

						'''simple docstring'''



						_snake_case						:					str    =   ""
						_snake_case						:					int
						_snake_case						:					int
						_snake_case						:					int

						for keychar, cipherchar in zip(cycle(__lowercase						) ,      __lowercase						):
												_snake_case						:					str    =   cipherchar ^ keychar
												if decodedchar not in VALID_INTS:
																		return None
												decoded += chr(__lowercase						)

						return decoded
def  snake_case						(__lowercase						)	->	list[str]:

						'''simple docstring'''



						_snake_case						:					list[str]    =   []
						for key in product(__lowercase ,      repeat=3						):
												_snake_case						:					Union[str, Any]    =   try_key(__lowercase ,      __lowercase						)
												if encoded is not None:
																		possibles.append(__lowercase						)
						return possibles
def  snake_case						(__lowercase ,      __lowercase						)	->	list[str]:

						'''simple docstring'''



						return [possible for possible in possibles if common_word in possible.lower()]
def  snake_case						(__lowercase = "p059_cipher.txt"						)	->	int:

						'''simple docstring'''



						_snake_case						:					list[int]
						_snake_case						:					list[str]
						_snake_case						:					str
						_snake_case						:					str
						_snake_case						:					str    =   Path(__lowercase						).parent.joinpath(__lowercase						).read_text(encoding="utf-8"						)

						_snake_case						:					Dict    =   [int(__lowercase						) for number in data.strip().split(","						)]

						_snake_case						:					Tuple    =   filter_valid_chars(__lowercase						)
						for common_word in COMMON_WORDS:
												_snake_case						:					Optional[int]    =   filter_common_word(__lowercase ,      __lowercase						)
												if len(__lowercase						) == 1:
																		break

						_snake_case						:					int    =   possibles[0]
						return sum(ord(__lowercase						) for char in decoded_text						)


if __name__ == "__main__":
				print(F'''{solution() = }''')