



import argparse
import json
import os
import sys
import tempfile
import unittest
from argparse import Namespace
from dataclasses import dataclass, field
from enum import Enum
from pathlib import Path
from typing import List, Literal, Optional

import yaml

from transformers import HfArgumentParser, TrainingArguments
from transformers.hf_argparser import make_choice_type_function, string_to_bool


# Since Python 3.10, we can use the builtin `|` operator for Union types
# See PEP 604: https://peps.python.org/pep-0604
snake_case    :						Union[str, Any]  =						sys.version_info >= (3, 1_0)
def 	snake_case__       (					__lowercase=None							,     __lowercase=None       )		->		Tuple:







						"""simple docstring"""

						return field(default_factory=lambda: default							,     metadata=__lowercase       )




@dataclass
class 							lowerCAmelCase__  :
					__A     :    int
					__A     :    float
					__A     :    str
					__A     :    bool




@dataclass
class 							lowerCAmelCase__  :
					__A     :    int           =						42
					__A     :    str           =						field(default='toto'					,							metadata={'help': 'help message'}						)




@dataclass
class 							lowerCAmelCase__  :
					__A     :    bool           =						False
					__A     :    bool           =						True
					__A     :    Optional[bool]           =						None




class 							lowerCAmelCase__  (	UpperCamelCase						):
					__A     :    List[Any]           =						'titi'
					__A     :    List[str]           =						'toto'




class 							lowerCAmelCase__  (	UpperCamelCase						):
					__A     :    Optional[int]           =						'titi'
					__A     :    Optional[Any]           =						'toto'
					__A     :    Optional[Any]           =						42




@dataclass
class 							lowerCAmelCase__  :
					__A     :    BasicEnum           =						"toto"







					def  _lowercase			(       self      :     int):
											A__	:							str						    = BasicEnum(self.foo)




@dataclass
class 							lowerCAmelCase__  :
					__A     :    MixedTypeEnum           =						"toto"







					def  _lowercase			(       self      :     str):
											A__	:							str						    = MixedTypeEnum(self.foo)




@dataclass
class 							lowerCAmelCase__  :
					__A     :    Optional[int]           =						None
					__A     :    Optional[float]           =						field(default=UpperCamelCase					,							metadata={'help': 'help message'}						)
					__A     :    Optional[str]           =						None
					__A     :    Optional[List[str]]           =						list_field(default=[]						)
					__A     :    Optional[List[int]]           =						list_field(default=[]						)




@dataclass
class 							lowerCAmelCase__  :
					__A     :    List[int]           =						list_field(default=[]						)
					__A     :    List[int]           =						list_field(default=[1, 2, 3]						)
					__A     :    List[str]           =						list_field(default=['Hallo', 'Bonjour', 'Hello']						)
					__A     :    List[float]           =						list_field(default=[0.1, 0.2, 0.3]						)




@dataclass
class 							lowerCAmelCase__  :
					__A     :    List[int]           =						field()
					__A     :    str           =						field()
					__A     :    BasicEnum           =						field()







					def  _lowercase			(       self      :     Dict):
											A__	:							Union[str, Any]						    = BasicEnum(self.required_enum)




@dataclass
class 							lowerCAmelCase__  :
					__A     :    int
					__A     :    "BasicEnum"           =						field()
					__A     :    "Optional[bool]"           =						None
					__A     :    "str"           =						field(default='toto'					,							metadata={'help': 'help message'}						)
					__A     :    "List[str]"           =						list_field(default=['Hallo', 'Bonjour', 'Hello']						)


if is_python_no_less_than_3_10:




		@dataclass
		class 							lowerCAmelCase__  :
							__A     :    bool           =						False
							__A     :    bool           =						True
							__A     :    bool | None           =						None




		@dataclass
		class 							lowerCAmelCase__  :
							__A     :    int | None           =						None
							__A     :    float | None           =						field(default=UpperCamelCase					,							metadata={'help': 'help message'}						)
							__A     :    str | None           =						None
							__A     :    list[str] | None           =						list_field(default=[]						)
							__A     :    list[int] | None           =						list_field(default=[]						)




class 							lowerCAmelCase__  (	unittest.TestCase						):







					def  _lowercase			(       self      :     List[Any]			,		_A      :     argparse.ArgumentParser			,		_A      :     argparse.ArgumentParser):
											self.assertEqual(len(a._actions)			,		len(b._actions))
											for x, y in zip(a._actions			,		b._actions):
																	A__	:							Union[str, Any]						    = {k: v for k, v in vars(_A).items() if k != "container"}
																	A__	:							Tuple						    = {k: v for k, v in vars(_A).items() if k != "container"}

																	# Choices with mixed type have custom function as "type"
																	# So we need to compare results directly for equality
																	if xx.get("choices"			,		_A) and yy.get("choices"			,		_A):
																							for expected_choice in yy["choices"] + xx["choices"]:
																													self.assertEqual(xx["type"](_A)			,		yy["type"](_A))
																							del xx["type"], yy["type"]

																	self.assertEqual(_A			,		_A)







					def  _lowercase			(       self      :     str):
											A__	:							Optional[Any]						    = HfArgumentParser(_A)

											A__	:							int						    = argparse.ArgumentParser()
											expected.add_argument("--foo"			,		type=_A			,		required=_A)
											expected.add_argument("--bar"			,		type=_A			,		required=_A)
											expected.add_argument("--baz"			,		type=_A			,		required=_A)
											expected.add_argument("--flag"			,		type=_A			,		default=_A			,		const=_A			,		nargs="?")
											self.argparsersEqual(_A			,		_A)

											A__	:							List[str]						    = ["--foo", "1", "--baz", "quux", "--bar", "0.5"]
											((A__)    ,       )	:							List[Any]						    = parser.parse_args_into_dataclasses(_A			,		look_for_args_file=_A)
											self.assertFalse(example.flag)







					def  _lowercase			(       self      :     str):
											A__	:							Optional[Any]						    = HfArgumentParser(_A)

											A__	:							str						    = argparse.ArgumentParser()
											expected.add_argument("--foo"			,		default=42			,		type=_A)
											expected.add_argument("--baz"			,		default="toto"			,		type=_A			,		help="help message")
											self.argparsersEqual(_A			,		_A)







					def  _lowercase			(       self      :     Optional[int]):
											A__	:							List[Any]						    = argparse.ArgumentParser()
											expected.add_argument("--foo"			,		type=_A			,		default=_A			,		const=_A			,		nargs="?")
											expected.add_argument("--baz"			,		type=_A			,		default=_A			,		const=_A			,		nargs="?")
											# A boolean no_* argument always has to come after its "default: True" regular counter-part
											# and its default must be set to False
											expected.add_argument("--no_baz"			,		action="store_false"			,		default=_A			,		dest="baz")
											expected.add_argument("--opt"			,		type=_A			,		default=_A)

											A__	:							str						    = [WithDefaultBoolExample]
											if is_python_no_less_than_3_10:
																	dataclass_types.append(_A)

											for dataclass_type in dataclass_types:
																	A__	:							Optional[int]						    = HfArgumentParser(_A)
																	self.argparsersEqual(_A			,		_A)

																	A__	:							Any						    = parser.parse_args([])
																	self.assertEqual(_A			,		Namespace(foo=_A			,		baz=_A			,		opt=_A))

																	A__	:							List[Any]						    = parser.parse_args(["--foo", "--no_baz"])
																	self.assertEqual(_A			,		Namespace(foo=_A			,		baz=_A			,		opt=_A))

																	A__	:							Optional[int]						    = parser.parse_args(["--foo", "--baz"])
																	self.assertEqual(_A			,		Namespace(foo=_A			,		baz=_A			,		opt=_A))

																	A__	:							List[Any]						    = parser.parse_args(["--foo", "True", "--baz", "True", "--opt", "True"])
																	self.assertEqual(_A			,		Namespace(foo=_A			,		baz=_A			,		opt=_A))

																	A__	:							Dict						    = parser.parse_args(["--foo", "False", "--baz", "False", "--opt", "False"])
																	self.assertEqual(_A			,		Namespace(foo=_A			,		baz=_A			,		opt=_A))







					def  _lowercase			(       self      :     Tuple):
											A__	:							Dict						    = HfArgumentParser(_A)

											A__	:							List[str]						    = argparse.ArgumentParser()
											expected.add_argument(
											    "--foo"			,		default="toto"			,		choices=["titi", "toto", 42]			,		type=make_choice_type_function(["titi", "toto", 42])			,		)
											self.argparsersEqual(_A			,		_A)

											A__	:							Optional[int]						    = parser.parse_args([])
											self.assertEqual(args.foo			,		"toto")
											A__	:							Optional[Any]						    = parser.parse_args_into_dataclasses([])[0]
											self.assertEqual(enum_ex.foo			,		MixedTypeEnum.toto)

											A__	:							str						    = parser.parse_args(["--foo", "titi"])
											self.assertEqual(args.foo			,		"titi")
											A__	:							Optional[Any]						    = parser.parse_args_into_dataclasses(["--foo", "titi"])[0]
											self.assertEqual(enum_ex.foo			,		MixedTypeEnum.titi)

											A__	:							int						    = parser.parse_args(["--foo", "42"])
											self.assertEqual(args.foo			,		42)
											A__	:							int						    = parser.parse_args_into_dataclasses(["--foo", "42"])[0]
											self.assertEqual(enum_ex.foo			,		MixedTypeEnum.fourtytwo)







					def  _lowercase			(       self      :     int):




											@dataclass
											class 							lowerCAmelCase__  :
																__A     :    Literal["titi", "toto", 42]           =						"toto"




											A__	:							Dict						    = HfArgumentParser(_A)

											A__	:							Tuple						    = argparse.ArgumentParser()
											expected.add_argument(
											    "--foo"			,		default="toto"			,		choices=("titi", "toto", 42)			,		type=make_choice_type_function(["titi", "toto", 42])			,		)
											self.argparsersEqual(_A			,		_A)

											A__	:							Optional[int]						    = parser.parse_args([])
											self.assertEqual(args.foo			,		"toto")

											A__	:							Union[str, Any]						    = parser.parse_args(["--foo", "titi"])
											self.assertEqual(args.foo			,		"titi")

											A__	:							Optional[int]						    = parser.parse_args(["--foo", "42"])
											self.assertEqual(args.foo			,		42)







					def  _lowercase			(       self      :     int):
											A__	:							List[str]						    = HfArgumentParser(_A)

											A__	:							int						    = argparse.ArgumentParser()
											expected.add_argument("--foo_int"			,		nargs="+"			,		default=[]			,		type=_A)
											expected.add_argument("--bar_int"			,		nargs="+"			,		default=[1, 2, 3]			,		type=_A)
											expected.add_argument("--foo_str"			,		nargs="+"			,		default=["Hallo", "Bonjour", "Hello"]			,		type=_A)
											expected.add_argument("--foo_float"			,		nargs="+"			,		default=[0.1, 0.2, 0.3]			,		type=_A)

											self.argparsersEqual(_A			,		_A)

											A__	:							Optional[Any]						    = parser.parse_args([])
											self.assertEqual(
											    _A			,		Namespace(foo_int=[]			,		bar_int=[1, 2, 3]			,		foo_str=["Hallo", "Bonjour", "Hello"]			,		foo_float=[0.1, 0.2, 0.3])			,		)

											A__	:							Union[str, Any]						    = parser.parse_args("--foo_int 1 --bar_int 2 3 --foo_str a b c --foo_float 0.1 0.7".split())
											self.assertEqual(_A			,		Namespace(foo_int=[1]			,		bar_int=[2, 3]			,		foo_str=["a", "b", "c"]			,		foo_float=[0.1, 0.7]))







					def  _lowercase			(       self      :     List[str]):
											A__	:							str						    = argparse.ArgumentParser()
											expected.add_argument("--foo"			,		default=_A			,		type=_A)
											expected.add_argument("--bar"			,		default=_A			,		type=_A			,		help="help message")
											expected.add_argument("--baz"			,		default=_A			,		type=_A)
											expected.add_argument("--ces"			,		nargs="+"			,		default=[]			,		type=_A)
											expected.add_argument("--des"			,		nargs="+"			,		default=[]			,		type=_A)

											A__	:							Dict						    = [OptionalExample]
											if is_python_no_less_than_3_10:
																	dataclass_types.append(_A)

											for dataclass_type in dataclass_types:
																	A__	:							int						    = HfArgumentParser(_A)

																	self.argparsersEqual(_A			,		_A)

																	A__	:							Optional[Any]						    = parser.parse_args([])
																	self.assertEqual(_A			,		Namespace(foo=_A			,		bar=_A			,		baz=_A			,		ces=[]			,		des=[]))

																	A__	:							Tuple						    = parser.parse_args("--foo 12 --bar 3.14 --baz 42 --ces a b c --des 1 2 3".split())
																	self.assertEqual(_A			,		Namespace(foo=12			,		bar=3.14			,		baz="42"			,		ces=["a", "b", "c"]			,		des=[1, 2, 3]))







					def  _lowercase			(       self      :     Union[str, Any]):
											A__	:							Optional[Any]						    = HfArgumentParser(_A)

											A__	:							Union[str, Any]						    = argparse.ArgumentParser()
											expected.add_argument("--required_list"			,		nargs="+"			,		type=_A			,		required=_A)
											expected.add_argument("--required_str"			,		type=_A			,		required=_A)
											expected.add_argument(
											    "--required_enum"			,		type=make_choice_type_function(["titi", "toto"])			,		choices=["titi", "toto"]			,		required=_A			,		)
											self.argparsersEqual(_A			,		_A)







					def  _lowercase			(       self      :     Any):
											A__	:							Dict						    = HfArgumentParser(_A)

											A__	:							Optional[Any]						    = argparse.ArgumentParser()
											expected.add_argument("--foo"			,		type=_A			,		required=_A)
											expected.add_argument(
											    "--required_enum"			,		type=make_choice_type_function(["titi", "toto"])			,		choices=["titi", "toto"]			,		required=_A			,		)
											expected.add_argument("--opt"			,		type=_A			,		default=_A)
											expected.add_argument("--baz"			,		default="toto"			,		type=_A			,		help="help message")
											expected.add_argument("--foo_str"			,		nargs="+"			,		default=["Hallo", "Bonjour", "Hello"]			,		type=_A)
											self.argparsersEqual(_A			,		_A)







					def  _lowercase			(       self      :     Optional[Any]):
											A__	:							List[str]						    = HfArgumentParser(_A)

											A__	:							Optional[Any]						    = {
											    "foo": 12,
											    "bar": 3.14,
											    "baz": "42",
											    "flag": True,
											}

											A__	:							int						    = parser.parse_dict(_A)[0]
											A__	:							Dict						    = BasicExample(**_A)
											self.assertEqual(_A			,		_A)







					def  _lowercase			(       self      :     Optional[int]):
											A__	:							Union[str, Any]						    = HfArgumentParser(_A)

											A__	:							List[str]						    = {
											    "foo": 12,
											    "bar": 3.14,
											    "baz": "42",
											    "flag": True,
											    "extra": 42,
											}

											self.assertRaises(_A			,		parser.parse_dict			,		_A			,		allow_extra_keys=_A)







					def  _lowercase			(       self      :     str):
											A__	:							Tuple						    = HfArgumentParser(_A)

											A__	:							Tuple						    = {
											    "foo": 12,
											    "bar": 3.14,
											    "baz": "42",
											    "flag": True,
											}
											with tempfile.TemporaryDirectory() as tmp_dir:
																	A__	:							Optional[int]						    = os.path.join(_A			,		"temp_json")
																	os.mkdir(_A)
																	with open(temp_local_path + ".json"			,		"w+") as f:
																							json.dump(_A			,		_A)
																	A__	:							List[str]						    = parser.parse_yaml_file(Path(temp_local_path + ".json"))[0]

											A__	:							int						    = BasicExample(**_A)
											self.assertEqual(_A			,		_A)







					def  _lowercase			(       self      :     Tuple):
											A__	:							Tuple						    = HfArgumentParser(_A)

											A__	:							Tuple						    = {
											    "foo": 12,
											    "bar": 3.14,
											    "baz": "42",
											    "flag": True,
											}
											with tempfile.TemporaryDirectory() as tmp_dir:
																	A__	:							int						    = os.path.join(_A			,		"temp_yaml")
																	os.mkdir(_A)
																	with open(temp_local_path + ".yaml"			,		"w+") as f:
																							yaml.dump(_A			,		_A)
																	A__	:							str						    = parser.parse_yaml_file(Path(temp_local_path + ".yaml"))[0]
											A__	:							Tuple						    = BasicExample(**_A)
											self.assertEqual(_A			,		_A)







					def  _lowercase			(       self      :     Optional[int]):
											A__	:							Optional[int]						    = HfArgumentParser(_A)
											self.assertIsNotNone(_A)




# Lint as: python3
import sys
from collections.abc import Mapping
from typing import TYPE_CHECKING, Dict, Optional

import numpy as np
import pyarrow as pa

from .. import config
from ..utils.logging import get_logger
from ..utils.py_utils import map_nested
from .formatting import TensorFormatter


if TYPE_CHECKING:
		import jax
		import jaxlib

snake_case    :						Optional[int]  =						get_logger()

snake_case    :						Optional[dict]  =						None




class 							lowerCAmelCase__  (	TensorFormatter[Mapping, 'jax.Array', Mapping]						):







					def __init__(       self      :     Optional[Any]			,		_A      :     Optional[int]=None			,		_A      :     str=None			,		**_A      :     Tuple):
											super().__init__(features=_A)
											import jax
											from jaxlib.xla_client import Device

											if isinstance(_A			,		_A):
																	raise ValueError(
																	    F'Expected {device} to be a `str` not {type(_A)}, as `jaxlib.xla_extension.Device` '
																	    "is not serializable neither with `pickle` nor with `dill`. Instead you can surround "
																	    "the device with `str()` to get its string identifier that will be internally mapped "
																	    "to the actual `jaxlib.xla_extension.Device`.")
											A__	:							Union[str, Any]						    = device if isinstance(_A			,		_A) else str(jax.devices()[0])
											# using global variable since `jaxlib.xla_extension.Device` is not serializable neither
											# with `pickle` nor with `dill`, so we need to use a global variable instead
											global DEVICE_MAPPING
											if DEVICE_MAPPING is None:
																	A__	:							Any						    = self._map_devices_to_str()
											if self.device not in list(DEVICE_MAPPING.keys()):
																	logger.warning(
																	    F'Device with string identifier {self.device} not listed among the available '
																	    F'devices: {list(DEVICE_MAPPING.keys())}, so falling back to the default '
																	    F'device: {str(jax.devices()[0])}.')
																	A__	:							str						    = str(jax.devices()[0])
											A__	:							List[Any]						    = jnp_array_kwargs







					@staticmethod
					def  _lowercase			(       ):
											import jax

											return {str(_A): device for device in jax.devices()}







					def  _lowercase			(       self      :     Tuple			,		_A      :     int):
											import jax
											import jax.numpy as jnp

											if isinstance(_A			,		_A) and column:
																	if all(
																	    isinstance(_A			,		jax.Array) and x.shape == column[0].shape and x.dtype == column[0].dtype for x in column):
																							return jnp.stack(_A			,		axis=0)
											return column







					def  _lowercase			(       self      :     List[str]			,		_A      :     Union[str, Any]):
											import jax
											import jax.numpy as jnp

											if isinstance(_A			,		(str, bytes, type(_A))):
																	return value
											elif isinstance(_A			,		(np.character, np.ndarray)) and np.issubdtype(value.dtype			,		np.character):
																	return value.tolist()

											A__	:							Dict						    = {}

											if isinstance(_A			,		(np.number, np.ndarray)) and np.issubdtype(value.dtype			,		np.integer):
																	# the default int precision depends on the jax config
																	# see https://jax.readthedocs.io/en/latest/notebooks/Common_Gotchas_in_JAX.html#double-64bit-precision
																	if jax.config.jax_enable_xaa:
																							A__	:							Union[str, Any]						    = {"dtype": jnp.intaa}
																	else:
																							A__	:							int						    = {"dtype": jnp.intaa}
											elif isinstance(_A			,		(np.number, np.ndarray)) and np.issubdtype(value.dtype			,		np.floating):
																	A__	:							str						    = {"dtype": jnp.floataa}
											elif config.PIL_AVAILABLE and "PIL" in sys.modules:
																	import PIL.Image

																	if isinstance(_A			,		PIL.Image.Image):
																							A__	:							Any						    = np.asarray(_A)

        # using global variable since `jaxlib.xla_extension.Device` is not serializable neither
        # with `pickle` nor with `dill`, so we need to use a global variable instead
											global DEVICE_MAPPING
											if DEVICE_MAPPING is None:
																	A__	:							Optional[Any]						    = self._map_devices_to_str()

											with jax.default_device(DEVICE_MAPPING[self.device]):
																	# calling jnp.array on a np.ndarray does copy the data
																	# see https://github.com/google/jax/issues/4486
																	return jnp.array(_A			,		**{**default_dtype, **self.jnp_array_kwargs})







					def  _lowercase			(       self      :     Dict			,		_A      :     Any):
											import jax

											# support for torch, tf, jax etc.
											if config.TORCH_AVAILABLE and "torch" in sys.modules:
																	import torch

																	if isinstance(_A			,		torch.Tensor):
																							return self._tensorize(data_struct.detach().cpu().numpy()[()])
											if hasattr(_A			,		"__array__") and not isinstance(_A			,		jax.Array):
																	A__	:							Any						    = data_struct.__array__()
											# support for nested types like struct of list of struct
											if isinstance(_A			,		np.ndarray):
																	if data_struct.dtype == object:  # jax arrays cannot be instantied from an array of objects
																							return self._consolidate([self.recursive_tensorize(_A) for substruct in data_struct])
											elif isinstance(_A			,		(list, tuple)):
																	return self._consolidate([self.recursive_tensorize(_A) for substruct in data_struct])
											return self._tensorize(_A)







					def  _lowercase			(       self      :     Optional[Any]			,		_A      :     dict):
											return map_nested(self._recursive_tensorize			,		_A			,		map_list=_A)







					def  _lowercase			(       self      :     List[str]			,		_A      :     pa.Table):
											A__	:							Dict						    = self.numpy_arrow_extractor().extract_row(_A)
											A__	:							int						    = self.python_features_decoder.decode_row(_A)
											return self.recursive_tensorize(_A)







					def  _lowercase			(       self      :     int			,		_A      :     pa.Table):
											A__	:							int						    = self.numpy_arrow_extractor().extract_column(_A)
											A__	:							Optional[int]						    = self.python_features_decoder.decode_column(_A			,		pa_table.column_names[0])
											A__	:							Dict						    = self.recursive_tensorize(_A)
											A__	:							List[Any]						    = self._consolidate(_A)
											return column







					def  _lowercase			(       self      :     Tuple			,		_A      :     pa.Table):
											A__	:							Any						    = self.numpy_arrow_extractor().extract_batch(_A)
											A__	:							str						    = self.python_features_decoder.decode_batch(_A)
											A__	:							Optional[Any]						    = self.recursive_tensorize(_A)
											for column_name in batch:
																	A__	:							Optional[int]						    = self._consolidate(batch[column_name])
											return batch