



"""simple docstring"""





import itertools
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa

import datasets
from datasets.table import table_cast


lowerCamelCase          =						datasets.utils.logging.get_logger(__name__)
@dataclass
class       lowercase__   (    datasets.BuilderConfig      ):







       '''simple docstring'''


       UpperCamelCase						=     None



class       lowercase__   (    datasets.ArrowBasedBuilder      ):







       '''simple docstring'''


       UpperCamelCase						=     ArrowConfig



       def      lowercase__				(							self			:  List[Any]    ) ->						int:


             '''simple docstring'''




             return datasets.DatasetInfo(features=self.config.features    )



       def      lowercase__				(							self			:  int		,				_UpperCAmelCase			:  List[str]    ) ->						Dict:


             '''simple docstring'''




             if not self.config.data_files:
                   raise ValueError(F"""At least one data file must be specified, but got data_files={self.config.data_files}"""    )
             UpperCAmelCase_			 =							dl_manager.download_and_extract(self.config.data_files    )
             if isinstance(lowercase_		,				(str, list, tuple)    ):
                   UpperCAmelCase_			 =							data_files
                   if isinstance(lowercase_		,				lowercase_    ):
                         UpperCAmelCase_			 =							[files]
                   # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
                   UpperCAmelCase_			 =							[dl_manager.iter_files(lowercase_    ) for file in files]
                   return [datasets.SplitGenerator(name=datasets.Split.TRAIN		,				gen_kwargs={"files": files}    )]
             UpperCAmelCase_			 =							[]
             for split_name, files in data_files.items():
                   if isinstance(lowercase_		,				lowercase_    ):
                         UpperCAmelCase_			 =							[files]
                   # Use `dl_manager.iter_files` to skip hidden files in an extracted archive
                   UpperCAmelCase_			 =							[dl_manager.iter_files(lowercase_    ) for file in files]
                   # Infer features is they are stoed in the arrow schema
                   if self.info.features is None:
                         for file in itertools.chain.from_iterable(lowercase_    ):
                               with open(lowercase_		,				"rb"    ) as f:
                                     UpperCAmelCase_			 =							datasets.Features.from_arrow_schema(pa.ipc.open_stream(lowercase_    ).schema    )
                               break
                   splits.append(datasets.SplitGenerator(name=lowercase_		,				gen_kwargs={"files": files}    )    )
             return splits



       def      lowercase__				(							self			:  int		,				_UpperCAmelCase			:  Tuple    ) ->						Tuple:


             '''simple docstring'''




             if self.info.features is not None:
                   # more expensive cast to support nested features with keys in a different order
                   # allows str <-> int/float or str to Audio for example
                   UpperCAmelCase_			 =							table_cast(lowercase_		,				self.info.features.arrow_schema    )
             return pa_table







       def      lowercase__				(							self			:  List[Any]		,				_UpperCAmelCase			:  Any    ) ->						List[str]:


             '''simple docstring'''




             for file_idx, file in enumerate(itertools.chain.from_iterable(lowercase_    )    ):
                   with open(lowercase_		,				"rb"    ) as f:
                         try:
                               for batch_idx, record_batch in enumerate(pa.ipc.open_stream(lowercase_    )    ):
                                     UpperCAmelCase_			 =							pa.Table.from_batches([record_batch]    )
                                     # Uncomment for debugging (will print the Arrow table size and elements)
                                     # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                                     # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                     yield F"""{file_idx}_{batch_idx}""", self._cast_table(lowercase_    )
                         except ValueError as e:
                               logger.error(F"""Failed to read file '{file}' with error {type(lowercase_    )}: {e}"""    )
                               raise





from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


__SCREAMING_SNAKE_CASE						:    str     = logging.get_logger(__name__)

__SCREAMING_SNAKE_CASE						:    Union[str, Any]     = {
    'nghuyong/ernie-1.0-base-zh': 'https://huggingface.co/nghuyong/ernie-1.0-base-zh/resolve/main/config.json',
    'nghuyong/ernie-2.0-base-en': 'https://huggingface.co/nghuyong/ernie-2.0-base-en/resolve/main/config.json',
    'nghuyong/ernie-2.0-large-en': 'https://huggingface.co/nghuyong/ernie-2.0-large-en/resolve/main/config.json',
    'nghuyong/ernie-3.0-base-zh': 'https://huggingface.co/nghuyong/ernie-3.0-base-zh/resolve/main/config.json',
    'nghuyong/ernie-3.0-medium-zh': 'https://huggingface.co/nghuyong/ernie-3.0-medium-zh/resolve/main/config.json',
    'nghuyong/ernie-3.0-mini-zh': 'https://huggingface.co/nghuyong/ernie-3.0-mini-zh/resolve/main/config.json',
    'nghuyong/ernie-3.0-micro-zh': 'https://huggingface.co/nghuyong/ernie-3.0-micro-zh/resolve/main/config.json',
    'nghuyong/ernie-3.0-nano-zh': 'https://huggingface.co/nghuyong/ernie-3.0-nano-zh/resolve/main/config.json',
    'nghuyong/ernie-gram-zh': 'https://huggingface.co/nghuyong/ernie-gram-zh/resolve/main/config.json',
    'nghuyong/ernie-health-zh': 'https://huggingface.co/nghuyong/ernie-health-zh/resolve/main/config.json',
}



class    lowercase_    (     __snake_case		):
							_lowerCamelCase						=		'ernie'



							def __init__(						self      ,				lowercase_=30_522      ,				lowercase_=768      ,				lowercase_=12      ,				lowercase_=12      ,				lowercase_=3_072      ,				lowercase_="gelu"      ,				lowercase_=0.1      ,				lowercase_=0.1      ,				lowercase_=512      ,				lowercase_=2      ,				lowercase_=3      ,				lowercase_=False      ,				lowercase_=0.02      ,				lowercase_=1e-12      ,				lowercase_=0      ,				lowercase_="absolute"      ,				lowercase_=True      ,				lowercase_=None      ,				**lowercase_      ,				):
													super().__init__(pad_token_id=lowercase_      ,				**lowercase_				)

													_snake_case						:     List[Any]				=    vocab_size
													_snake_case						:     Dict				=    hidden_size
													_snake_case						:     Any				=    num_hidden_layers
													_snake_case						:     List[str]				=    num_attention_heads
													_snake_case						:     int				=    hidden_act
													_snake_case						:     Tuple				=    intermediate_size
													_snake_case						:     Tuple				=    hidden_dropout_prob
													_snake_case						:     Optional[int]				=    attention_probs_dropout_prob
													_snake_case						:     Any				=    max_position_embeddings
													_snake_case						:     List[str]				=    type_vocab_size
													_snake_case						:     List[Any]				=    task_type_vocab_size
													_snake_case						:     Optional[int]				=    use_task_id
													_snake_case						:     Tuple				=    initializer_range
													_snake_case						:     Union[str, Any]				=    layer_norm_eps
													_snake_case						:     Optional[Any]				=    position_embedding_type
													_snake_case						:     int				=    use_cache
													_snake_case						:     Optional[Any]				=    classifier_dropout




class    lowercase_    (     __snake_case		):



							@property
							def        UpperCamelCase       (						self				):
													if self.task == "multiple-choice":
																			_snake_case						:     str				=    {0: "batch", 1: "choice", 2: "sequence"}
													else:
																			_snake_case						:     Optional[int]				=    {0: "batch", 1: "sequence"}
													return OrderedDict(
													    [
													        ("input_ids", dynamic_axis),
													        ("attention_mask", dynamic_axis),
													        ("token_type_ids", dynamic_axis),
													        ("task_type_ids", dynamic_axis),
													    ]				)