



import argparse
import json
import os
import sys
import tempfile
import unittest
from argparse import Namespace
from dataclasses import dataclass, field
from enum import Enum
from pathlib import Path
from typing import List, Literal, Optional

import yaml

from transformers import HfArgumentParser, TrainingArguments
from transformers.hf_argparser import make_choice_type_function, string_to_bool


# Since Python 3.10, we can use the builtin `|` operator for Union types
# See PEP 604: https://peps.python.org/pep-0604
__snake_case     :List[Any]			       =sys.version_info >= (3, 10)






def  lowerCamelCase_	(      lowerCAmelCase__	:	List[Any]=None			,	lowerCAmelCase__	:	Optional[int]=None				)       ->							List[Any]:

							'''simple docstring'''






							return field(default_factory=lambda: default			,	metadata=lowerCAmelCase__				)



@dataclass
class        lowerCAmelCase__					:
				A_   :     int
				A_   :     float
				A_   :     str
				A_   :     bool





@dataclass
class        lowerCAmelCase__					:
				A_   :     int								=     4_2
				A_   :     str								=     field(default='toto' ,				metadata={'help': 'help message'}       )





@dataclass
class        lowerCAmelCase__					:
				A_   :     bool								=     False
				A_   :     bool								=     True
				A_   :     Optional[bool]								=     None





class        lowerCAmelCase__					(  _lowerCamelCase       ):
				A_   :     Union[str, Any]								=     'titi'
				A_   :     Tuple								=     'toto'





class        lowerCAmelCase__					(  _lowerCamelCase       ):
				A_   :     List[Any]								=     'titi'
				A_   :     List[Any]								=     'toto'
				A_   :     Union[str, Any]								=     4_2





@dataclass
class        lowerCAmelCase__					:
				A_   :     BasicEnum								=     "toto"


				def __UpperCamelCase						(							self      :  List[Any]       )		->     Optional[Any]:
											A      						=    BasicEnum(self.foo       )





@dataclass
class        lowerCAmelCase__					:
				A_   :     MixedTypeEnum								=     "toto"


				def __UpperCamelCase						(							self      :  Any       )		->     Tuple:
											A      						=    MixedTypeEnum(self.foo       )





@dataclass
class        lowerCAmelCase__					:
				A_   :     Optional[int]								=     None
				A_   :     Optional[float]								=     field(default=_lowerCamelCase ,				metadata={'help': 'help message'}       )
				A_   :     Optional[str]								=     None
				A_   :     Optional[List[str]]								=     list_field(default=[]       )
				A_   :     Optional[List[int]]								=     list_field(default=[]       )





@dataclass
class        lowerCAmelCase__					:
				A_   :     List[int]								=     list_field(default=[]       )
				A_   :     List[int]								=     list_field(default=[1, 2, 3]       )
				A_   :     List[str]								=     list_field(default=['Hallo', 'Bonjour', 'Hello']       )
				A_   :     List[float]								=     list_field(default=[0.1, 0.2, 0.3]       )





@dataclass
class        lowerCAmelCase__					:
				A_   :     List[int]								=     field()
				A_   :     str								=     field()
				A_   :     BasicEnum								=     field()


				def __UpperCamelCase						(							self      :  str       )		->     Dict:
											A      						=    BasicEnum(self.required_enum       )





@dataclass
class        lowerCAmelCase__					:
				A_   :     int
				A_   :     "BasicEnum"								=     field()
				A_   :     "Optional[bool]"								=     None
				A_   :     "str"								=     field(default='toto' ,				metadata={'help': 'help message'}       )
				A_   :     "List[str]"								=     list_field(default=['Hallo', 'Bonjour', 'Hello']       )


if is_python_no_less_than_3_10:



				@dataclass
				class        lowerCAmelCase__					:
								A_   :     bool								=     False
								A_   :     bool								=     True
								A_   :     bool | None								=     None





				@dataclass
				class        lowerCAmelCase__					:
								A_   :     int | None								=     None
								A_   :     float | None								=     field(default=_lowerCamelCase ,				metadata={'help': 'help message'}       )
								A_   :     str | None								=     None
								A_   :     list[str] | None								=     list_field(default=[]       )
								A_   :     list[int] | None								=     list_field(default=[]       )



class        lowerCAmelCase__					(  unittest.TestCase       ):


				def __UpperCamelCase						(							self      :  Union[str, Any]						, __UpperCamelCase      :  argparse.ArgumentParser						, __UpperCamelCase      :  argparse.ArgumentParser       )		->     List[str]:
											self.assertEqual(len(a._actions       )						, len(b._actions       )       )
											for x, y in zip(a._actions						, b._actions       ):
																		A      						=    {k: v for k, v in vars(__UpperCamelCase       ).items() if k != 'container'}
																		A      						=    {k: v for k, v in vars(__UpperCamelCase       ).items() if k != 'container'}

																		# Choices with mixed type have custom function as "type"
																		# So we need to compare results directly for equality
																		if xx.get('choices'						, __UpperCamelCase       ) and yy.get('choices'						, __UpperCamelCase       ):
																									for expected_choice in yy["choices"] + xx["choices"]:
																																self.assertEqual(xx['type'](__UpperCamelCase       )						, yy['type'](__UpperCamelCase       )       )
																									del xx["type"], yy["type"]

																		self.assertEqual(__UpperCamelCase						, __UpperCamelCase       )


				def __UpperCamelCase						(							self      :  List[Any]       )		->     str:
											A      						=    HfArgumentParser(__UpperCamelCase       )

											A      						=    argparse.ArgumentParser()
											expected.add_argument('--foo'						, type=__UpperCamelCase						, required=__UpperCamelCase       )
											expected.add_argument('--bar'						, type=__UpperCamelCase						, required=__UpperCamelCase       )
											expected.add_argument('--baz'						, type=__UpperCamelCase						, required=__UpperCamelCase       )
											expected.add_argument('--flag'						, type=__UpperCamelCase						, default=__UpperCamelCase						, const=__UpperCamelCase						, nargs='?'       )
											self.argparsersEqual(__UpperCamelCase						, __UpperCamelCase       )

											A      						=    ['--foo', '1', '--baz', 'quux', '--bar', '0.5']
											((A)						,   )      						=    parser.parse_args_into_dataclasses(__UpperCamelCase						, look_for_args_file=__UpperCamelCase       )
											self.assertFalse(example.flag       )


				def __UpperCamelCase						(							self      :  Optional[int]       )		->     Optional[Any]:
											A      						=    HfArgumentParser(__UpperCamelCase       )

											A      						=    argparse.ArgumentParser()
											expected.add_argument('--foo'						, default=42						, type=__UpperCamelCase       )
											expected.add_argument('--baz'						, default='toto'						, type=__UpperCamelCase						, help='help message'       )
											self.argparsersEqual(__UpperCamelCase						, __UpperCamelCase       )


				def __UpperCamelCase						(							self      :  Union[str, Any]       )		->     str:
											A      						=    argparse.ArgumentParser()
											expected.add_argument('--foo'						, type=__UpperCamelCase						, default=__UpperCamelCase						, const=__UpperCamelCase						, nargs='?'       )
											expected.add_argument('--baz'						, type=__UpperCamelCase						, default=__UpperCamelCase						, const=__UpperCamelCase						, nargs='?'       )
											# A boolean no_* argument always has to come after its "default: True" regular counter-part
											# and its default must be set to False
											expected.add_argument('--no_baz'						, action='store_false'						, default=__UpperCamelCase						, dest='baz'       )
											expected.add_argument('--opt'						, type=__UpperCamelCase						, default=__UpperCamelCase       )

											A      						=    [WithDefaultBoolExample]
											if is_python_no_less_than_3_10:
																		dataclass_types.append(__UpperCamelCase       )

											for dataclass_type in dataclass_types:
																		A      						=    HfArgumentParser(__UpperCamelCase       )
																		self.argparsersEqual(__UpperCamelCase						, __UpperCamelCase       )

																		A      						=    parser.parse_args([]       )
																		self.assertEqual(__UpperCamelCase						, Namespace(foo=__UpperCamelCase						, baz=__UpperCamelCase						, opt=__UpperCamelCase       )       )

																		A      						=    parser.parse_args(['--foo', '--no_baz']       )
																		self.assertEqual(__UpperCamelCase						, Namespace(foo=__UpperCamelCase						, baz=__UpperCamelCase						, opt=__UpperCamelCase       )       )

																		A      						=    parser.parse_args(['--foo', '--baz']       )
																		self.assertEqual(__UpperCamelCase						, Namespace(foo=__UpperCamelCase						, baz=__UpperCamelCase						, opt=__UpperCamelCase       )       )

																		A      						=    parser.parse_args(['--foo', 'True', '--baz', 'True', '--opt', 'True']       )
																		self.assertEqual(__UpperCamelCase						, Namespace(foo=__UpperCamelCase						, baz=__UpperCamelCase						, opt=__UpperCamelCase       )       )

																		A      						=    parser.parse_args(['--foo', 'False', '--baz', 'False', '--opt', 'False']       )
																		self.assertEqual(__UpperCamelCase						, Namespace(foo=__UpperCamelCase						, baz=__UpperCamelCase						, opt=__UpperCamelCase       )       )


				def __UpperCamelCase						(							self      :  Any       )		->     Optional[int]:
											A      						=    HfArgumentParser(__UpperCamelCase       )

											A      						=    argparse.ArgumentParser()
											expected.add_argument(
											    '--foo'						, default='toto'						, choices=['titi', 'toto', 42]						, type=make_choice_type_function(['titi', 'toto', 42]       )						, )
											self.argparsersEqual(__UpperCamelCase						, __UpperCamelCase       )

											A      						=    parser.parse_args([]       )
											self.assertEqual(args.foo						, 'toto'       )
											A      						=    parser.parse_args_into_dataclasses([]       )[0]
											self.assertEqual(enum_ex.foo						, MixedTypeEnum.toto       )

											A      						=    parser.parse_args(['--foo', 'titi']       )
											self.assertEqual(args.foo						, 'titi'       )
											A      						=    parser.parse_args_into_dataclasses(['--foo', 'titi']       )[0]
											self.assertEqual(enum_ex.foo						, MixedTypeEnum.titi       )

											A      						=    parser.parse_args(['--foo', '42']       )
											self.assertEqual(args.foo						, 42       )
											A      						=    parser.parse_args_into_dataclasses(['--foo', '42']       )[0]
											self.assertEqual(enum_ex.foo						, MixedTypeEnum.fourtytwo       )


				def __UpperCamelCase						(							self      :  Any       )		->     List[Any]:



											@dataclass
											class        lowerCAmelCase__					:
															A_   :     Literal["titi", "toto", 4_2]								=     "toto"





											A      						=    HfArgumentParser(__UpperCamelCase       )

											A      						=    argparse.ArgumentParser()
											expected.add_argument(
											    '--foo'						, default='toto'						, choices=('titi', 'toto', 42)						, type=make_choice_type_function(['titi', 'toto', 42]       )						, )
											self.argparsersEqual(__UpperCamelCase						, __UpperCamelCase       )

											A      						=    parser.parse_args([]       )
											self.assertEqual(args.foo						, 'toto'       )

											A      						=    parser.parse_args(['--foo', 'titi']       )
											self.assertEqual(args.foo						, 'titi'       )

											A      						=    parser.parse_args(['--foo', '42']       )
											self.assertEqual(args.foo						, 42       )


				def __UpperCamelCase						(							self      :  List[Any]       )		->     Any:
											A      						=    HfArgumentParser(__UpperCamelCase       )

											A      						=    argparse.ArgumentParser()
											expected.add_argument('--foo_int'						, nargs='+'						, default=[]						, type=__UpperCamelCase       )
											expected.add_argument('--bar_int'						, nargs='+'						, default=[1, 2, 3]						, type=__UpperCamelCase       )
											expected.add_argument('--foo_str'						, nargs='+'						, default=['Hallo', 'Bonjour', 'Hello']						, type=__UpperCamelCase       )
											expected.add_argument('--foo_float'						, nargs='+'						, default=[0.1, 0.2, 0.3]						, type=__UpperCamelCase       )

											self.argparsersEqual(__UpperCamelCase						, __UpperCamelCase       )

											A      						=    parser.parse_args([]       )
											self.assertEqual(
											    __UpperCamelCase						, Namespace(foo_int=[]						, bar_int=[1, 2, 3]						, foo_str=['Hallo', 'Bonjour', 'Hello']						, foo_float=[0.1, 0.2, 0.3]       )						, )

											A      						=    parser.parse_args('--foo_int 1 --bar_int 2 3 --foo_str a b c --foo_float 0.1 0.7'.split()       )
											self.assertEqual(__UpperCamelCase						, Namespace(foo_int=[1]						, bar_int=[2, 3]						, foo_str=['a', 'b', 'c']						, foo_float=[0.1, 0.7]       )       )


				def __UpperCamelCase						(							self      :  Optional[Any]       )		->     List[str]:
											A      						=    argparse.ArgumentParser()
											expected.add_argument('--foo'						, default=__UpperCamelCase						, type=__UpperCamelCase       )
											expected.add_argument('--bar'						, default=__UpperCamelCase						, type=__UpperCamelCase						, help='help message'       )
											expected.add_argument('--baz'						, default=__UpperCamelCase						, type=__UpperCamelCase       )
											expected.add_argument('--ces'						, nargs='+'						, default=[]						, type=__UpperCamelCase       )
											expected.add_argument('--des'						, nargs='+'						, default=[]						, type=__UpperCamelCase       )

											A      						=    [OptionalExample]
											if is_python_no_less_than_3_10:
																		dataclass_types.append(__UpperCamelCase       )

											for dataclass_type in dataclass_types:
																		A      						=    HfArgumentParser(__UpperCamelCase       )

																		self.argparsersEqual(__UpperCamelCase						, __UpperCamelCase       )

																		A      						=    parser.parse_args([]       )
																		self.assertEqual(__UpperCamelCase						, Namespace(foo=__UpperCamelCase						, bar=__UpperCamelCase						, baz=__UpperCamelCase						, ces=[]						, des=[]       )       )

																		A      						=    parser.parse_args('--foo 12 --bar 3.14 --baz 42 --ces a b c --des 1 2 3'.split()       )
																		self.assertEqual(__UpperCamelCase						, Namespace(foo=12						, bar=3.1_4						, baz='42'						, ces=['a', 'b', 'c']						, des=[1, 2, 3]       )       )


				def __UpperCamelCase						(							self      :  Dict       )		->     Tuple:
											A      						=    HfArgumentParser(__UpperCamelCase       )

											A      						=    argparse.ArgumentParser()
											expected.add_argument('--required_list'						, nargs='+'						, type=__UpperCamelCase						, required=__UpperCamelCase       )
											expected.add_argument('--required_str'						, type=__UpperCamelCase						, required=__UpperCamelCase       )
											expected.add_argument(
											    '--required_enum'						, type=make_choice_type_function(['titi', 'toto']       )						, choices=['titi', 'toto']						, required=__UpperCamelCase						, )
											self.argparsersEqual(__UpperCamelCase						, __UpperCamelCase       )


				def __UpperCamelCase						(							self      :  Any       )		->     List[str]:
											A      						=    HfArgumentParser(__UpperCamelCase       )

											A      						=    argparse.ArgumentParser()
											expected.add_argument('--foo'						, type=__UpperCamelCase						, required=__UpperCamelCase       )
											expected.add_argument(
											    '--required_enum'						, type=make_choice_type_function(['titi', 'toto']       )						, choices=['titi', 'toto']						, required=__UpperCamelCase						, )
											expected.add_argument('--opt'						, type=__UpperCamelCase						, default=__UpperCamelCase       )
											expected.add_argument('--baz'						, default='toto'						, type=__UpperCamelCase						, help='help message'       )
											expected.add_argument('--foo_str'						, nargs='+'						, default=['Hallo', 'Bonjour', 'Hello']						, type=__UpperCamelCase       )
											self.argparsersEqual(__UpperCamelCase						, __UpperCamelCase       )


				def __UpperCamelCase						(							self      :  List[str]       )		->     int:
											A      						=    HfArgumentParser(__UpperCamelCase       )

											A      						=    {
											    'foo': 12,
											    'bar': 3.1_4,
											    'baz': '42',
											    'flag': True,
											}

											A      						=    parser.parse_dict(__UpperCamelCase       )[0]
											A      						=    BasicExample(**__UpperCamelCase       )
											self.assertEqual(__UpperCamelCase						, __UpperCamelCase       )


				def __UpperCamelCase						(							self      :  List[Any]       )		->     Tuple:
											A      						=    HfArgumentParser(__UpperCamelCase       )

											A      						=    {
											    'foo': 12,
											    'bar': 3.1_4,
											    'baz': '42',
											    'flag': True,
											    'extra': 42,
											}

											self.assertRaises(__UpperCamelCase						, parser.parse_dict						, __UpperCamelCase						, allow_extra_keys=__UpperCamelCase       )


				def __UpperCamelCase						(							self      :  Optional[int]       )		->     List[str]:
											A      						=    HfArgumentParser(__UpperCamelCase       )

											A      						=    {
											    'foo': 12,
											    'bar': 3.1_4,
											    'baz': '42',
											    'flag': True,
											}
											with tempfile.TemporaryDirectory() as tmp_dir:
																		A      						=    os.path.join(__UpperCamelCase						, 'temp_json'       )
																		os.mkdir(__UpperCamelCase       )
																		with open(temp_local_path + '.json'						, 'w+'       ) as f:
																									json.dump(__UpperCamelCase						, __UpperCamelCase       )
																		A      						=    parser.parse_yaml_file(Path(temp_local_path + '.json'       )       )[0]

											A      						=    BasicExample(**__UpperCamelCase       )
											self.assertEqual(__UpperCamelCase						, __UpperCamelCase       )


				def __UpperCamelCase						(							self      :  int       )		->     Any:
											A      						=    HfArgumentParser(__UpperCamelCase       )

											A      						=    {
											    'foo': 12,
											    'bar': 3.1_4,
											    'baz': '42',
											    'flag': True,
											}
											with tempfile.TemporaryDirectory() as tmp_dir:
																		A      						=    os.path.join(__UpperCamelCase						, 'temp_yaml'       )
																		os.mkdir(__UpperCamelCase       )
																		with open(temp_local_path + '.yaml'						, 'w+'       ) as f:
																									yaml.dump(__UpperCamelCase						, __UpperCamelCase       )
																		A      						=    parser.parse_yaml_file(Path(temp_local_path + '.yaml'       )       )[0]
											A      						=    BasicExample(**__UpperCamelCase       )
											self.assertEqual(__UpperCamelCase						, __UpperCamelCase       )


				def __UpperCamelCase						(							self      :  List[str]       )		->     Tuple:
											A      						=    HfArgumentParser(__UpperCamelCase       )
											self.assertIsNotNone(__UpperCamelCase       )




from typing import Callable, Optional

from .. import Features
from ..packaged_modules.generator.generator import Generator
from .abc import AbstractDatasetInputStream



class        lowerCAmelCase__					(  _lowerCamelCase       ):


				def __init__(							self      :  int						, __UpperCamelCase      :  Callable						, __UpperCamelCase      :  Optional[Features] = None						, __UpperCamelCase      :  str = None						, __UpperCamelCase      :  bool = False						, __UpperCamelCase      :  bool = False						, __UpperCamelCase      :  Optional[dict] = None						, __UpperCamelCase      :  Optional[int] = None						, **__UpperCamelCase      :  str						, )		->     List[str]:
											super().__init__(
											    features=__UpperCamelCase						, cache_dir=__UpperCamelCase						, keep_in_memory=__UpperCamelCase						, streaming=__UpperCamelCase						, num_proc=__UpperCamelCase						, **__UpperCamelCase						, )
											A      						=    Generator(
											    cache_dir=__UpperCamelCase						, features=__UpperCamelCase						, generator=__UpperCamelCase						, gen_kwargs=__UpperCamelCase						, **__UpperCamelCase						, )


				def __UpperCamelCase						(							self      :  Union[str, Any]       )		->     Tuple:
											# Build iterable dataset
											if self.streaming:
																		A      						=    self.builder.as_streaming_dataset(split='train'       )
											# Build regular (map-style) dataset
											else:
																		A      						=    None
																		A      						=    None
																		A      						=    None
																		A      						=    None

																		self.builder.download_and_prepare(
																		    download_config=__UpperCamelCase						, download_mode=__UpperCamelCase						, verification_mode=__UpperCamelCase						, base_path=__UpperCamelCase						, num_proc=self.num_proc						, )
																		A      						=    self.builder.as_dataset(
																		    split='train'						, verification_mode=__UpperCamelCase						, in_memory=self.keep_in_memory       )
											return dataset