




'''simple docstring'''







import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


lowercase      :   Tuple      =       datasets.utils.logging.get_logger(__name__)







@dataclass
class 					_a (datasets.BuilderConfig	):





     '''simple docstring'''



     lowerCAmelCase_  :     int          =					None
     lowerCAmelCase_  :     Optional[Any]          =					"""utf-8"""
     lowerCAmelCase_  :     Dict          =					None
     lowerCAmelCase_  :     str          =					None
     lowerCAmelCase_  :     Tuple          =					True  # deprecated
     lowerCAmelCase_  :     int          =					None  # deprecated
     lowerCAmelCase_  :     List[Any]          =					10 << 20  # 10MB
     lowerCAmelCase_  :     Any          =					None







class 					_a (datasets.ArrowBasedBuilder	):





     '''simple docstring'''



     lowerCAmelCase_  :     Tuple          =					JsonConfig







     def 			snake_case_      (				self							)      ->				Any:
          if self.config.block_size is not None:
               logger.warning("""The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead"""							)
               snake_case			:      Union[str, Any]        =			self.config.block_size
          if self.config.use_threads is not True:
               logger.warning(
                   """The JSON loader parameter `use_threads` is deprecated and doesn't have any effect anymore."""							)
          if self.config.newlines_in_values is not None:
               raise ValueError("""The JSON loader parameter `newlines_in_values` is no longer supported"""							)
          return datasets.DatasetInfo(features=self.config.features							)







     def 			snake_case_      (				self							,__a							)      ->				Dict:
          if not self.config.data_files:
               raise ValueError(F'''At least one data file must be specified, but got data_files={self.config.data_files}'''							)
          snake_case			:      int        =			dl_manager.download_and_extract(self.config.data_files							)
          if isinstance(__a							,(str, list, tuple)							):
               snake_case			:      Union[str, Any]        =			data_files
               if isinstance(__a							,__a							):
                    snake_case			:      Union[str, Any]        =			[files]
               snake_case			:      Dict        =			[dl_manager.iter_files(__a							) for file in files]
               return [datasets.SplitGenerator(name=datasets.Split.TRAIN							,gen_kwargs={"""files""": files}							)]
          snake_case			:      Optional[int]        =			[]
          for split_name, files in data_files.items():
               if isinstance(__a							,__a							):
                    snake_case			:      Union[str, Any]        =			[files]
               snake_case			:      int        =			[dl_manager.iter_files(__a							) for file in files]
               splits.append(datasets.SplitGenerator(name=__a							,gen_kwargs={"""files""": files}							)							)
          return splits







     def 			snake_case_      (				self							,__a							)      ->				pa.Table:
          if self.config.features is not None:
               # adding missing columns
               for column_name in set(self.config.features							) - set(pa_table.column_names							):
                    snake_case			:      List[str]        =			self.config.features.arrow_schema.field(__a							).type
                    snake_case			:      Optional[int]        =			pa_table.append_column(__a							,pa.array([None] * len(__a							)							,type=__a							)							)
               # more expensive cast to support nested structures with keys in a different order
               # allows str <-> int/float or str to Audio for example
               snake_case			:      str        =			table_cast(__a							,self.config.features.arrow_schema							)
          return pa_table







     def 			snake_case_      (				self							,__a							)      ->				str:
          for file_idx, file in enumerate(itertools.chain.from_iterable(__a							)							):
               # If the file is one json object and if we need to look at the list of items in one specific field
               if self.config.field is not None:
                    with open(__a							,encoding=self.config.encoding							,errors=self.config.encoding_errors							) as f:
                         snake_case			:      str        =			json.load(__a							)

                    # We keep only the field we are interested in
                    snake_case			:      Tuple        =			dataset[self.config.field]

                    # We accept two format: a list of dicts or a dict of lists
                    if isinstance(__a							,(list, tuple)							):
                         snake_case			:      Tuple        =			set().union(*[row.keys() for row in dataset]							)
                         snake_case			:      List[Any]        =			{col: [row.get(__a							) for row in dataset] for col in keys}
                    else:
                         snake_case			:      List[str]        =			dataset
                    snake_case			:      int        =			pa.Table.from_pydict(__a							)
                    yield file_idx, self._cast_table(__a							)

               # If the file has one json object per line
               else:
                    with open(__a							,"""rb"""							) as f:
                         snake_case			:      Union[str, Any]        =			0
                         # Use block_size equal to the chunk size divided by 32 to leverage multithreading
                         # Set a default minimum value of 16kB if the chunk size is really small
                         snake_case			:      Tuple        =			max(self.config.chunksize // 32							,16 << 10							)
                         snake_case			:      str        =			(
                             self.config.encoding_errors if self.config.encoding_errors is not None else """strict"""
                         )
                         while True:
                              snake_case			:      Union[str, Any]        =			f.read(self.config.chunksize							)
                              if not batch:
                                   break
                              # Finish current line
                              try:
                                   batch += f.readline()
                              except (AttributeError, io.UnsupportedOperation):
                                   batch += readline(__a							)
                              # PyArrow only accepts utf-8 encoded bytes
                              if self.config.encoding != "utf-8":
                                   snake_case			:      Optional[int]        =			batch.decode(self.config.encoding							,errors=__a							).encode("""utf-8"""							)
                              try:
                                   while True:
                                        try:
                                             snake_case			:      Union[str, Any]        =			paj.read_json(
                                                 io.BytesIO(__a							)							,read_options=paj.ReadOptions(block_size=__a							)							)
                                             break
                                        except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
                                             if (
                                                 isinstance(__a							,pa.ArrowInvalid							)
                                                 and "straddling" not in str(__a							)
                                                 or block_size > len(__a							)
                                             ):
                                                  raise
                                             else:
                                                  # Increase the block size in case it was too small.
                                                  # The block size will be reset for the next file.
                                                  logger.debug(
                                                      F'''Batch of {len(__a							)} bytes couldn\'t be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}.'''							)
                                                  block_size *= 2
                              except pa.ArrowInvalid as e:
                                   try:
                                        with open(
                                            __a							,encoding=self.config.encoding							,errors=self.config.encoding_errors							) as f:
                                             snake_case			:      Optional[int]        =			json.load(__a							)
                                   except json.JSONDecodeError:
                                        logger.error(F'''Failed to read file \'{file}\' with error {type(__a							)}: {e}'''							)
                                        raise e
                                   # If possible, parse the file as a list of json objects and exit the loop
                                   if isinstance(__a							,__a							):  # list is the only sequence type supported in JSON
                                        try:
                                             snake_case			:      int        =			set().union(*[row.keys() for row in dataset]							)
                                             snake_case			:      List[Any]        =			{col: [row.get(__a							) for row in dataset] for col in keys}
                                             snake_case			:      Union[str, Any]        =			pa.Table.from_pydict(__a							)
                                        except (pa.ArrowInvalid, AttributeError) as e:
                                             logger.error(F'''Failed to read file \'{file}\' with error {type(__a							)}: {e}'''							)
                                             raise ValueError(F'''Not able to read records in the JSON file at {file}.'''							) from None
                                        yield file_idx, self._cast_table(__a							)
                                        break
                                   else:
                                        logger.error(F'''Failed to read file \'{file}\' with error {type(__a							)}: {e}'''							)
                                        raise ValueError(
                                            F'''Not able to read records in the JSON file at {file}. '''
                                            F'''You should probably indicate the field of the JSON file containing your records. '''
                                            F'''This JSON file contain the following fields: {str(list(dataset.keys()							)							)}. '''
                                            F'''Select the correct one and provide it as `field=\'XXX\'` to the dataset loading method. '''							) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                              yield (file_idx, batch_idx), self._cast_table(__a							)
                              batch_idx += 1











from collections import OrderedDict
from typing import List, Mapping

from packaging import version

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


__a:     Dict   					=					logging.get_logger(__name__)

__a:     Optional[int]   					=					{
    '''google/efficientnet-b7''': '''https://huggingface.co/google/efficientnet-b7/resolve/main/config.json''',
}
class 		SCREAMING_SNAKE_CASE__	(  UpperCAmelCase     ):

  '''simple docstring'''







  _lowerCamelCase    =  '''efficientnet'''


  def __init__(      self		:			Dict       ,				lowerCamelCase		:			int = 3       ,				lowerCamelCase		:			int = 600       ,				lowerCamelCase		:			float = 2.0       ,				lowerCamelCase		:			float = 3.1       ,				lowerCamelCase		:			int = 8       ,				lowerCamelCase		:			List[int] = [3, 3, 5, 3, 5, 5, 3]       ,				lowerCamelCase		:			List[int] = [32, 16, 24, 40, 80, 112, 192]       ,				lowerCamelCase		:			List[int] = [16, 24, 40, 80, 112, 192, 320]       ,				lowerCamelCase		:			List[int] = []       ,				lowerCamelCase		:			List[int] = [1, 2, 2, 2, 1, 2, 1]       ,				lowerCamelCase		:			List[int] = [1, 2, 2, 3, 3, 4, 1]       ,				lowerCamelCase		:			List[int] = [1, 6, 6, 6, 6, 6, 6]       ,				lowerCamelCase		:			float = 0.25       ,				lowerCamelCase		:			str = "swish"       ,				lowerCamelCase		:			int = 2560       ,				lowerCamelCase		:			str = "mean"       ,				lowerCamelCase		:			float = 0.02       ,				lowerCamelCase		:			float = 0.001       ,				lowerCamelCase		:			float = 0.99       ,				lowerCamelCase		:			float = 0.5       ,				lowerCamelCase		:			float = 0.2       ,				**lowerCamelCase		:			List[str]       ,				)						->	Union[str, Any]:


         """simple docstring"""
         super().__init__(**lowerCamelCase							)

         _UpperCAmelCase							=    num_channels
         _UpperCAmelCase							=    image_size
         _UpperCAmelCase							=    width_coefficient
         _UpperCAmelCase							=    depth_coefficient
         _UpperCAmelCase							=    depth_divisor
         _UpperCAmelCase							=    kernel_sizes
         _UpperCAmelCase							=    in_channels
         _UpperCAmelCase							=    out_channels
         _UpperCAmelCase							=    depthwise_padding
         _UpperCAmelCase							=    strides
         _UpperCAmelCase							=    num_block_repeats
         _UpperCAmelCase							=    expand_ratios
         _UpperCAmelCase							=    squeeze_expansion_ratio
         _UpperCAmelCase							=    hidden_act
         _UpperCAmelCase							=    hidden_dim
         _UpperCAmelCase							=    pooling_type
         _UpperCAmelCase							=    initializer_range
         _UpperCAmelCase							=    batch_norm_eps
         _UpperCAmelCase							=    batch_norm_momentum
         _UpperCAmelCase							=    dropout_rate
         _UpperCAmelCase							=    drop_connect_rate
         _UpperCAmelCase							=    sum(lowerCamelCase							) * 4



class 		SCREAMING_SNAKE_CASE__	(  UpperCAmelCase     ):

  '''simple docstring'''







  _lowerCamelCase    =  version.parse('''1.11'''     )


  @property
  def  lowerCamelCase       (      self		:			Optional[Any]							)						->	Mapping[str, Mapping[int, str]]:


         """simple docstring"""
         return OrderedDict(
             [
                 ("""pixel_values""", {0: """batch""", 1: """num_channels""", 2: """height""", 3: """width"""}),
             ]							)





  @property
  def  lowerCamelCase       (      self		:			Dict							)						->	float:


         """simple docstring"""
         return 1E-5