


import argparse
import json
import os
import sys
import tempfile
import unittest
from argparse import Namespace
from dataclasses import dataclass, field
from enum import Enum
from pathlib import Path
from typing import List, Literal, Optional

import yaml

from transformers import HfArgumentParser, TrainingArguments
from transformers.hf_argparser import make_choice_type_function, string_to_bool


# Since Python 3.10, we can use the builtin `|` operator for Union types
# See PEP 604: https://peps.python.org/pep-0604
_lowerCAmelCase:						Any   				=       sys.version_info >= (3, 10)






def   _lowercase(					__a				:						int=None    ,     __a				:						Any=None     ):
							return field(default_factory=lambda: default    ,     metadata=__a     )




@dataclass
class       lowercase_					:
				snake_case	    =42
				snake_case	    =42
				snake_case	    =42
				snake_case	    =42







@dataclass
class       lowercase_					:
				snake_case	    =42
				snake_case	    =field(default='toto'      ,					metadata={'help': 'help message'}   )







@dataclass
class       lowercase_					:
				snake_case	    =False
				snake_case	    =True
				snake_case	    =None







class       lowercase_					(lowercase__   ):
				snake_case	    ='titi'
				snake_case	    ='toto'







class       lowercase_					(lowercase__   ):
				snake_case	    ='titi'
				snake_case	    ='toto'
				snake_case	    =42







@dataclass
class       lowercase_					:
				snake_case	    ="toto"





				def 						__UpperCamelCase    (				self)  -> List[str]:
											a__      =BasicEnum(self.foo)







@dataclass
class       lowercase_					:
				snake_case	    ="toto"





				def 						__UpperCamelCase    (				self)  -> List[str]:
											a__      =MixedTypeEnum(self.foo)







@dataclass
class       lowercase_					:
				snake_case	    =None
				snake_case	    =field(default=lowercase__      ,					metadata={'help': 'help message'}   )
				snake_case	    =None
				snake_case	    =list_field(default=[]   )
				snake_case	    =list_field(default=[]   )







@dataclass
class       lowercase_					:
				snake_case	    =list_field(default=[]   )
				snake_case	    =list_field(default=[1, 2, 3]   )
				snake_case	    =list_field(default=['Hallo', 'Bonjour', 'Hello']   )
				snake_case	    =list_field(default=[0.1, 0.2, 0.3]   )







@dataclass
class       lowercase_					:
				snake_case	    =field()
				snake_case	    =field()
				snake_case	    =field()





				def 						__UpperCamelCase    (				self)  -> List[Any]:
											a__      =BasicEnum(self.required_enum)







@dataclass
class       lowercase_					:
				snake_case	    =42
				snake_case	    =field()
				snake_case	    =None
				snake_case	    =field(default='toto'      ,					metadata={'help': 'help message'}   )
				snake_case	    =list_field(default=['Hallo', 'Bonjour', 'Hello']   )


if is_python_no_less_than_3_10:




	@dataclass
	class       lowercase_					:
					snake_case	    =False
					snake_case	    =True
					snake_case	    =None







	@dataclass
	class       lowercase_					:
					snake_case	    =None
					snake_case	    =field(default=lowercase__      ,					metadata={'help': 'help message'}   )
					snake_case	    =None
					snake_case	    =list_field(default=[]   )
					snake_case	    =list_field(default=[]   )







class       lowercase_					(unittest.TestCase   ):





				def 						__UpperCamelCase    (				self       ,  lowercase_       ,  lowercase_)  -> int:
											self.assertEqual(len(a._actions)       ,  len(b._actions))
											for x, y in zip(a._actions       ,  b._actions):
																		a__      ={k: v for k, v in vars(lowercase_).items() if k != 'container'}
																		a__      ={k: v for k, v in vars(lowercase_).items() if k != 'container'}

																		# Choices with mixed type have custom function as "type"
																		# So we need to compare results directly for equality
																		if xx.get('choices'       ,  lowercase_) and yy.get('choices'       ,  lowercase_):
																									for expected_choice in yy["choices"] + xx["choices"]:
																																self.assertEqual(xx['type'](lowercase_)       ,  yy['type'](lowercase_))
																									del xx["type"], yy["type"]

																		self.assertEqual(lowercase_       ,  lowercase_)





				def 						__UpperCamelCase    (				self)  -> List[Any]:
											a__      =HfArgumentParser(lowercase_)

											a__      =argparse.ArgumentParser()
											expected.add_argument('--foo'       ,  type=lowercase_       ,  required=lowercase_)
											expected.add_argument('--bar'       ,  type=lowercase_       ,  required=lowercase_)
											expected.add_argument('--baz'       ,  type=lowercase_       ,  required=lowercase_)
											expected.add_argument('--flag'       ,  type=lowercase_       ,  default=lowercase_       ,  const=lowercase_       ,  nargs='?')
											self.argparsersEqual(lowercase_       ,  lowercase_)

											a__      =['--foo', '1', '--baz', 'quux', '--bar', '0.5']
											((a__)    ,     )      =parser.parse_args_into_dataclasses(lowercase_       ,  look_for_args_file=lowercase_)
											self.assertFalse(example.flag)





				def 						__UpperCamelCase    (				self)  -> Union[str, Any]:
											a__      =HfArgumentParser(lowercase_)

											a__      =argparse.ArgumentParser()
											expected.add_argument('--foo'       ,  default=42       ,  type=lowercase_)
											expected.add_argument('--baz'       ,  default='toto'       ,  type=lowercase_       ,  help='help message')
											self.argparsersEqual(lowercase_       ,  lowercase_)





				def 						__UpperCamelCase    (				self)  -> Optional[Any]:
											a__      =argparse.ArgumentParser()
											expected.add_argument('--foo'       ,  type=lowercase_       ,  default=lowercase_       ,  const=lowercase_       ,  nargs='?')
											expected.add_argument('--baz'       ,  type=lowercase_       ,  default=lowercase_       ,  const=lowercase_       ,  nargs='?')
											# A boolean no_* argument always has to come after its "default: True" regular counter-part
											# and its default must be set to False
											expected.add_argument('--no_baz'       ,  action='store_false'       ,  default=lowercase_       ,  dest='baz')
											expected.add_argument('--opt'       ,  type=lowercase_       ,  default=lowercase_)

											a__      =[WithDefaultBoolExample]
											if is_python_no_less_than_3_10:
																		dataclass_types.append(lowercase_)

											for dataclass_type in dataclass_types:
																		a__      =HfArgumentParser(lowercase_)
																		self.argparsersEqual(lowercase_       ,  lowercase_)

																		a__      =parser.parse_args([])
																		self.assertEqual(lowercase_       ,  Namespace(foo=lowercase_       ,  baz=lowercase_       ,  opt=lowercase_))

																		a__      =parser.parse_args(['--foo', '--no_baz'])
																		self.assertEqual(lowercase_       ,  Namespace(foo=lowercase_       ,  baz=lowercase_       ,  opt=lowercase_))

																		a__      =parser.parse_args(['--foo', '--baz'])
																		self.assertEqual(lowercase_       ,  Namespace(foo=lowercase_       ,  baz=lowercase_       ,  opt=lowercase_))

																		a__      =parser.parse_args(['--foo', 'True', '--baz', 'True', '--opt', 'True'])
																		self.assertEqual(lowercase_       ,  Namespace(foo=lowercase_       ,  baz=lowercase_       ,  opt=lowercase_))

																		a__      =parser.parse_args(['--foo', 'False', '--baz', 'False', '--opt', 'False'])
																		self.assertEqual(lowercase_       ,  Namespace(foo=lowercase_       ,  baz=lowercase_       ,  opt=lowercase_))





				def 						__UpperCamelCase    (				self)  -> str:
											a__      =HfArgumentParser(lowercase_)

											a__      =argparse.ArgumentParser()
											expected.add_argument(
											    '--foo'       ,  default='toto'       ,  choices=['titi', 'toto', 42]       ,  type=make_choice_type_function(['titi', 'toto', 42])       ,  )
											self.argparsersEqual(lowercase_       ,  lowercase_)

											a__      =parser.parse_args([])
											self.assertEqual(args.foo       ,  'toto')
											a__      =parser.parse_args_into_dataclasses([])[0]
											self.assertEqual(enum_ex.foo       ,  MixedTypeEnum.toto)

											a__      =parser.parse_args(['--foo', 'titi'])
											self.assertEqual(args.foo       ,  'titi')
											a__      =parser.parse_args_into_dataclasses(['--foo', 'titi'])[0]
											self.assertEqual(enum_ex.foo       ,  MixedTypeEnum.titi)

											a__      =parser.parse_args(['--foo', '42'])
											self.assertEqual(args.foo       ,  42)
											a__      =parser.parse_args_into_dataclasses(['--foo', '42'])[0]
											self.assertEqual(enum_ex.foo       ,  MixedTypeEnum.fourtytwo)





				def 						__UpperCamelCase    (				self)  -> List[Any]:




											@dataclass
											class       lowercase_					:
															snake_case	    ="toto"







											a__      =HfArgumentParser(lowercase_)

											a__      =argparse.ArgumentParser()
											expected.add_argument(
											    '--foo'       ,  default='toto'       ,  choices=('titi', 'toto', 42)       ,  type=make_choice_type_function(['titi', 'toto', 42])       ,  )
											self.argparsersEqual(lowercase_       ,  lowercase_)

											a__      =parser.parse_args([])
											self.assertEqual(args.foo       ,  'toto')

											a__      =parser.parse_args(['--foo', 'titi'])
											self.assertEqual(args.foo       ,  'titi')

											a__      =parser.parse_args(['--foo', '42'])
											self.assertEqual(args.foo       ,  42)





				def 						__UpperCamelCase    (				self)  -> Optional[int]:
											a__      =HfArgumentParser(lowercase_)

											a__      =argparse.ArgumentParser()
											expected.add_argument('--foo_int'       ,  nargs='+'       ,  default=[]       ,  type=lowercase_)
											expected.add_argument('--bar_int'       ,  nargs='+'       ,  default=[1, 2, 3]       ,  type=lowercase_)
											expected.add_argument('--foo_str'       ,  nargs='+'       ,  default=['Hallo', 'Bonjour', 'Hello']       ,  type=lowercase_)
											expected.add_argument('--foo_float'       ,  nargs='+'       ,  default=[0.1, 0.2, 0.3]       ,  type=lowercase_)

											self.argparsersEqual(lowercase_       ,  lowercase_)

											a__      =parser.parse_args([])
											self.assertEqual(
											    lowercase_       ,  Namespace(foo_int=[]       ,  bar_int=[1, 2, 3]       ,  foo_str=['Hallo', 'Bonjour', 'Hello']       ,  foo_float=[0.1, 0.2, 0.3])       ,  )

											a__      =parser.parse_args('--foo_int 1 --bar_int 2 3 --foo_str a b c --foo_float 0.1 0.7'.split())
											self.assertEqual(lowercase_       ,  Namespace(foo_int=[1]       ,  bar_int=[2, 3]       ,  foo_str=['a', 'b', 'c']       ,  foo_float=[0.1, 0.7]))





				def 						__UpperCamelCase    (				self)  -> Dict:
											a__      =argparse.ArgumentParser()
											expected.add_argument('--foo'       ,  default=lowercase_       ,  type=lowercase_)
											expected.add_argument('--bar'       ,  default=lowercase_       ,  type=lowercase_       ,  help='help message')
											expected.add_argument('--baz'       ,  default=lowercase_       ,  type=lowercase_)
											expected.add_argument('--ces'       ,  nargs='+'       ,  default=[]       ,  type=lowercase_)
											expected.add_argument('--des'       ,  nargs='+'       ,  default=[]       ,  type=lowercase_)

											a__      =[OptionalExample]
											if is_python_no_less_than_3_10:
																		dataclass_types.append(lowercase_)

											for dataclass_type in dataclass_types:
																		a__      =HfArgumentParser(lowercase_)

																		self.argparsersEqual(lowercase_       ,  lowercase_)

																		a__      =parser.parse_args([])
																		self.assertEqual(lowercase_       ,  Namespace(foo=lowercase_       ,  bar=lowercase_       ,  baz=lowercase_       ,  ces=[]       ,  des=[]))

																		a__      =parser.parse_args('--foo 12 --bar 3.14 --baz 42 --ces a b c --des 1 2 3'.split())
																		self.assertEqual(lowercase_       ,  Namespace(foo=12       ,  bar=3.14       ,  baz='42'       ,  ces=['a', 'b', 'c']       ,  des=[1, 2, 3]))





				def 						__UpperCamelCase    (				self)  -> str:
											a__      =HfArgumentParser(lowercase_)

											a__      =argparse.ArgumentParser()
											expected.add_argument('--required_list'       ,  nargs='+'       ,  type=lowercase_       ,  required=lowercase_)
											expected.add_argument('--required_str'       ,  type=lowercase_       ,  required=lowercase_)
											expected.add_argument(
											    '--required_enum'       ,  type=make_choice_type_function(['titi', 'toto'])       ,  choices=['titi', 'toto']       ,  required=lowercase_       ,  )
											self.argparsersEqual(lowercase_       ,  lowercase_)





				def 						__UpperCamelCase    (				self)  -> str:
											a__      =HfArgumentParser(lowercase_)

											a__      =argparse.ArgumentParser()
											expected.add_argument('--foo'       ,  type=lowercase_       ,  required=lowercase_)
											expected.add_argument(
											    '--required_enum'       ,  type=make_choice_type_function(['titi', 'toto'])       ,  choices=['titi', 'toto']       ,  required=lowercase_       ,  )
											expected.add_argument('--opt'       ,  type=lowercase_       ,  default=lowercase_)
											expected.add_argument('--baz'       ,  default='toto'       ,  type=lowercase_       ,  help='help message')
											expected.add_argument('--foo_str'       ,  nargs='+'       ,  default=['Hallo', 'Bonjour', 'Hello']       ,  type=lowercase_)
											self.argparsersEqual(lowercase_       ,  lowercase_)





				def 						__UpperCamelCase    (				self)  -> List[Any]:
											a__      =HfArgumentParser(lowercase_)

											a__      ={
											    'foo': 12,
											    'bar': 3.14,
											    'baz': '42',
											    'flag': True,
											}

											a__      =parser.parse_dict(lowercase_)[0]
											a__      =BasicExample(**lowercase_)
											self.assertEqual(lowercase_       ,  lowercase_)





				def 						__UpperCamelCase    (				self)  -> Union[str, Any]:
											a__      =HfArgumentParser(lowercase_)

											a__      ={
											    'foo': 12,
											    'bar': 3.14,
											    'baz': '42',
											    'flag': True,
											    'extra': 42,
											}

											self.assertRaises(lowercase_       ,  parser.parse_dict       ,  lowercase_       ,  allow_extra_keys=lowercase_)





				def 						__UpperCamelCase    (				self)  -> List[Any]:
											a__      =HfArgumentParser(lowercase_)

											a__      ={
											    'foo': 12,
											    'bar': 3.14,
											    'baz': '42',
											    'flag': True,
											}
											with tempfile.TemporaryDirectory() as tmp_dir:
																		a__      =os.path.join(lowercase_       ,  'temp_json')
																		os.mkdir(lowercase_)
																		with open(temp_local_path + '.json'       ,  'w+') as f:
																									json.dump(lowercase_       ,  lowercase_)
																		a__      =parser.parse_yaml_file(Path(temp_local_path + '.json'))[0]

											a__      =BasicExample(**lowercase_)
											self.assertEqual(lowercase_       ,  lowercase_)





				def 						__UpperCamelCase    (				self)  -> Any:
											a__      =HfArgumentParser(lowercase_)

											a__      ={
											    'foo': 12,
											    'bar': 3.14,
											    'baz': '42',
											    'flag': True,
											}
											with tempfile.TemporaryDirectory() as tmp_dir:
																		a__      =os.path.join(lowercase_       ,  'temp_yaml')
																		os.mkdir(lowercase_)
																		with open(temp_local_path + '.yaml'       ,  'w+') as f:
																									yaml.dump(lowercase_       ,  lowercase_)
																		a__      =parser.parse_yaml_file(Path(temp_local_path + '.yaml'))[0]
											a__      =BasicExample(**lowercase_)
											self.assertEqual(lowercase_       ,  lowercase_)





				def 						__UpperCamelCase    (				self)  -> Union[str, Any]:
											a__      =HfArgumentParser(lowercase_)
											self.assertIsNotNone(lowercase_)








import itertools
from dataclasses import dataclass
from typing import List, Optional

import pyarrow as pa
import pyarrow.parquet as pq

import datasets
from datasets.table import table_cast


_UpperCAmelCase							:			Optional[Any]        =							datasets.utils.logging.get_logger(__name__)

@dataclass
class   lowerCAmelCase				(   datasets.BuilderConfig     ):
 UpperCAmelCase__						=						1_00_00
 UpperCAmelCase__						=						None
 UpperCAmelCase__						=						None


class   lowerCAmelCase				(   datasets.ArrowBasedBuilder     ):
 UpperCAmelCase__						=						ParquetConfig
 def 	A_      ( self    : int      )     ->      Optional[int]:
     return datasets.DatasetInfo(features=self.config.features      )
 def 	A_      ( self    : int     ,	UpperCAmelCase    : str      )     ->      Any:
     if not self.config.data_files:
         raise ValueError(F"""At least one data file must be specified, but got data_files={self.config.data_files}"""      )
     lowerCamelCase__       : List[Any]   					=						dl_manager.download_and_extract(self.config.data_files      )
     if isinstance(UpperCAmelCase     ,	(str, list, tuple)      ):
         lowerCamelCase__       : int   					=						data_files
         if isinstance(UpperCAmelCase     ,	UpperCAmelCase      ):
             lowerCamelCase__       : int   					=						[files]
         # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
         lowerCamelCase__       : str   					=						[dl_manager.iter_files(UpperCAmelCase      ) for file in files]
         return [datasets.SplitGenerator(name=datasets.Split.TRAIN     ,	gen_kwargs={'files': files}      )]
     lowerCamelCase__       : Optional[Any]   					=						[]
     for split_name, files in data_files.items():
         if isinstance(UpperCAmelCase     ,	UpperCAmelCase      ):
             lowerCamelCase__       : Optional[Any]   					=						[files]
         # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
         lowerCamelCase__       : Dict   					=						[dl_manager.iter_files(UpperCAmelCase      ) for file in files]
         # Infer features is they are stoed in the arrow schema
         if self.info.features is None:
             for file in itertools.chain.from_iterable(UpperCAmelCase      ):
                 with open(UpperCAmelCase     ,	'rb'      ) as f:
                     lowerCamelCase__       : Optional[Any]   					=						datasets.Features.from_arrow_schema(pq.read_schema(UpperCAmelCase      )      )
                 break
         splits.append(datasets.SplitGenerator(name=UpperCAmelCase     ,	gen_kwargs={'files': files}      )      )
     return splits
 def 	A_      ( self    : Optional[int]     ,	UpperCAmelCase    : pa.Table      )     ->      pa.Table:
     if self.info.features is not None:
         # more expensive cast to support nested features with keys in a different order
         # allows str <-> int/float or str to Audio for example
         lowerCamelCase__       : Any   					=						table_cast(UpperCAmelCase     ,	self.info.features.arrow_schema      )
     return pa_table







 def 	A_      ( self    : Any     ,	UpperCAmelCase    : Tuple      )     ->      List[str]:
     lowerCamelCase__       : Optional[Any]   					=						self.info.features.arrow_schema if self.info.features is not None else None
     if self.info.features is not None and self.config.columns is not None:
         if sorted(field.name for field in schema      ) != sorted(self.config.columns      ):
             raise ValueError(
                 F"""Tried to load parquet data with columns '{self.config.columns}' with mismatching features '{self.info.features}'"""      )
     for file_idx, file in enumerate(itertools.chain.from_iterable(UpperCAmelCase      )      ):
         with open(UpperCAmelCase     ,	'rb'      ) as f:
             lowerCamelCase__       : List[Any]   					=						pq.ParquetFile(UpperCAmelCase      )
             try:
                 for batch_idx, record_batch in enumerate(
                     parquet_file.iter_batches(batch_size=self.config.batch_size     ,	columns=self.config.columns      )      ):
                     lowerCamelCase__       : Tuple   					=						pa.Table.from_batches([record_batch]      )
                     # Uncomment for debugging (will print the Arrow table size and elements)
                     # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                     # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                     yield F"""{file_idx}_{batch_idx}""", self._cast_table(UpperCAmelCase      )
             except ValueError as e:
                 logger.error(F"""Failed to read file '{file}' with error {type(UpperCAmelCase      )}: {e}"""      )
                 raise





