





import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


lowerCAmelCase_				= datasets.utils.logging.get_logger(__name__)

@dataclass
class 					A     (datasets.BuilderConfig					):
      _SCREAMING_SNAKE_CASE       =				None
      _SCREAMING_SNAKE_CASE       =				"""utf-8"""
      _SCREAMING_SNAKE_CASE       =				None
      _SCREAMING_SNAKE_CASE       =				None
      _SCREAMING_SNAKE_CASE       =				True  # deprecated
      _SCREAMING_SNAKE_CASE       =				None  # deprecated
      _SCREAMING_SNAKE_CASE       =				10 << 20  # 10MB
      _SCREAMING_SNAKE_CASE       =				None


class 					A     (datasets.ArrowBasedBuilder					):
      _SCREAMING_SNAKE_CASE       =				JsonConfig



      def  __a			(       self       )   -> Tuple:



             '''simple docstring'''

             if self.config.block_size is not None:
                    logger.warning('''The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead'''       )
                    _snake_case   :				int  					=    self.config.block_size
             if self.config.use_threads is not True:
                    logger.warning(
                        '''The JSON loader parameter `use_threads` is deprecated and doesn\'t have any effect anymore.'''       )
             if self.config.newlines_in_values is not None:
                    raise ValueError('''The JSON loader parameter `newlines_in_values` is no longer supported'''       )
             return datasets.DatasetInfo(features=self.config.features       )



      def  __a			(       self  ,	lowercase_       )   -> Any:



             '''simple docstring'''

             if not self.config.data_files:
                    raise ValueError(F'''At least one data file must be specified, but got data_files={self.config.data_files}'''       )
             _snake_case   :				Tuple  					=    dl_manager.download_and_extract(self.config.data_files       )
             if isinstance(lowerCamelCase__  ,	(str, list, tuple)       ):
                    _snake_case   :				str  					=    data_files
                    if isinstance(lowerCamelCase__  ,	lowerCamelCase__       ):
                           _snake_case   :				str  					=    [files]
                    _snake_case   :				int  					=    [dl_manager.iter_files(lowerCamelCase__       ) for file in files]
                    return [datasets.SplitGenerator(name=datasets.Split.TRAIN  ,	gen_kwargs={'''files''': files}       )]
             _snake_case   :				List[Any]  					=    []
             for split_name, files in data_files.items():
                    if isinstance(lowerCamelCase__  ,	lowerCamelCase__       ):
                           _snake_case   :				Dict  					=    [files]
                    _snake_case   :				List[str]  					=    [dl_manager.iter_files(lowerCamelCase__       ) for file in files]
                    splits.append(datasets.SplitGenerator(name=lowerCamelCase__  ,	gen_kwargs={'''files''': files}       )       )
             return splits



      def  __a			(       self  ,	lowercase_       )   -> List[Any]:



             '''simple docstring'''

             if self.config.features is not None:
                    # adding missing columns
                    for column_name in set(self.config.features       ) - set(pa_table.column_names       ):
                           _snake_case   :				Optional[Any]  					=    self.config.features.arrow_schema.field(lowerCamelCase__       ).type
                           _snake_case   :				int  					=    pa_table.append_column(lowerCamelCase__  ,	pa.array([None] * len(lowerCamelCase__       )  ,	type=lowerCamelCase__       )       )
                    # more expensive cast to support nested structures with keys in a different order
                    # allows str <-> int/float or str to Audio for example
                    _snake_case   :				List[Any]  					=    table_cast(lowerCamelCase__  ,	self.config.features.arrow_schema       )
             return pa_table



      def  __a			(       self  ,	lowercase_       )   -> Optional[int]:



             '''simple docstring'''

             for file_idx, file in enumerate(itertools.chain.from_iterable(lowerCamelCase__       )       ):
                    # If the file is one json object and if we need to look at the list of items in one specific field
                    if self.config.field is not None:
                           with open(lowerCamelCase__  ,	encoding=self.config.encoding  ,	errors=self.config.encoding_errors       ) as f:
                                  _snake_case   :				str  					=    json.load(lowerCamelCase__       )

                           # We keep only the field we are interested in
                           _snake_case   :				Tuple  					=    dataset[self.config.field]

                           # We accept two format: a list of dicts or a dict of lists
                           if isinstance(lowerCamelCase__  ,	(list, tuple)       ):
                                  _snake_case   :				Optional[Any]  					=    set().union(*[row.keys() for row in dataset]       )
                                  _snake_case   :				List[str]  					=    {col: [row.get(lowerCamelCase__       ) for row in dataset] for col in keys}
                           else:
                                  _snake_case   :				Union[str, Any]  					=    dataset
                           _snake_case   :				Union[str, Any]  					=    pa.Table.from_pydict(lowerCamelCase__       )
                           yield file_idx, self._cast_table(lowerCamelCase__       )

                    # If the file has one json object per line
                    else:
                           with open(lowerCamelCase__  ,	'''rb'''       ) as f:
                                  _snake_case   :				Any  					=    0
                                  # Use block_size equal to the chunk size divided by 32 to leverage multithreading
                                  # Set a default minimum value of 16kB if the chunk size is really small
                                  _snake_case   :				int  					=    max(self.config.chunksize // 32  ,	16 << 10       )
                                  _snake_case   :				str  					=    (
                                      self.config.encoding_errors if self.config.encoding_errors is not None else "strict"
                                  )
                                  while True:
                                         _snake_case   :				str  					=    f.read(self.config.chunksize       )
                                         if not batch:
                                                break
                                         # Finish current line
                                         try:
                                                batch += f.readline()
                                         except (AttributeError, io.UnsupportedOperation):
                                                batch += readline(lowerCamelCase__       )
                                         # PyArrow only accepts utf-8 encoded bytes
                                         if self.config.encoding != "utf-8":
                                                _snake_case   :				List[str]  					=    batch.decode(self.config.encoding  ,	errors=lowerCamelCase__       ).encode('''utf-8'''       )
                                         try:
                                                while True:
                                                       try:
                                                              _snake_case   :				str  					=    paj.read_json(
                                                                  io.BytesIO(lowerCamelCase__       )  ,	read_options=paj.ReadOptions(block_size=lowerCamelCase__       )       )
                                                              break
                                                       except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
                                                              if (
                                                                  isinstance(lowerCamelCase__  ,	pa.ArrowInvalid       )
                                                                  and "straddling" not in str(lowerCamelCase__       )
                                                                  or block_size > len(lowerCamelCase__       )
                                                              ):
                                                                     raise
                                                              else:
                                                                     # Increase the block size in case it was too small.
                                                                     # The block size will be reset for the next file.
                                                                     logger.debug(
                                                                         F'''Batch of {len(lowerCamelCase__       )} bytes couldn\'t be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}.'''       )
                                                                     block_size *= 2
                                         except pa.ArrowInvalid as e:
                                                try:
                                                       with open(
                                                           lowerCamelCase__  ,	encoding=self.config.encoding  ,	errors=self.config.encoding_errors       ) as f:
                                                              _snake_case   :				int  					=    json.load(lowerCamelCase__       )
                                                except json.JSONDecodeError:
                                                       logger.error(F'''Failed to read file \'{file}\' with error {type(lowerCamelCase__       )}: {e}'''       )
                                                       raise e
                                                # If possible, parse the file as a list of json objects and exit the loop
                                                if isinstance(lowerCamelCase__  ,	lowerCamelCase__       ):  # list is the only sequence type supported in JSON
                                                       try:
                                                              _snake_case   :				int  					=    set().union(*[row.keys() for row in dataset]       )
                                                              _snake_case   :				int  					=    {col: [row.get(lowerCamelCase__       ) for row in dataset] for col in keys}
                                                              _snake_case   :				Tuple  					=    pa.Table.from_pydict(lowerCamelCase__       )
                                                       except (pa.ArrowInvalid, AttributeError) as e:
                                                              logger.error(F'''Failed to read file \'{file}\' with error {type(lowerCamelCase__       )}: {e}'''       )
                                                              raise ValueError(F'''Not able to read records in the JSON file at {file}.'''       ) from None
                                                       yield file_idx, self._cast_table(lowerCamelCase__       )
                                                       break
                                                else:
                                                       logger.error(F'''Failed to read file \'{file}\' with error {type(lowerCamelCase__       )}: {e}'''       )
                                                       raise ValueError(
                                                           F'''Not able to read records in the JSON file at {file}. '''
                                                           F'''You should probably indicate the field of the JSON file containing your records. '''
                                                           F'''This JSON file contain the following fields: {str(list(dataset.keys()       )       )}. '''
                                                           F'''Select the correct one and provide it as `field=\'XXX\'` to the dataset loading method. '''       ) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                                         yield (file_idx, batch_idx), self._cast_table(lowerCamelCase__       )
                                         batch_idx += 1









import importlib
import os
from dataclasses import dataclass
from enum import Enum
from typing import Any, Dict, Optional, Union

import torch

from ..utils import BaseOutput


_lowerCAmelCase  :      int			 ="""scheduler_config.json"""

class __UpperCamelCase     (				_a		):






					'''simple docstring'''



					__magic_name__        =   1
					__magic_name__        =   2
					__magic_name__        =   3
					__magic_name__        =   4
					__magic_name__        =   5
					__magic_name__        =   6
					__magic_name__        =   7
					__magic_name__        =   8
					__magic_name__        =   9
					__magic_name__        =   1_0
					__magic_name__        =   1_1
					__magic_name__        =   1_2
					__magic_name__        =   1_3
					__magic_name__        =   1_4

@dataclass
class __UpperCamelCase     (				_a		):






					'''simple docstring'''



					__magic_name__        =   42

class __UpperCamelCase     :






					'''simple docstring'''



					__magic_name__        =   SCHEDULER_CONFIG_NAME
					__magic_name__        =   []
					__magic_name__        =   True





					@classmethod
					def    _UpperCAmelCase    (				cls				,      lowerCamelCase__ = None				,      lowerCamelCase__ = None				,      lowerCamelCase__=False				,      **lowerCamelCase__				,      ):
						UpperCAmelCase__							,						UpperCAmelCase__							,						UpperCAmelCase__:	Tuple									= cls.load_config(
						    pretrained_model_name_or_path=lowerCamelCase__				,      subfolder=lowerCamelCase__				,      return_unused_kwargs=lowerCamelCase__				,      return_commit_hash=lowerCamelCase__				,      **lowerCamelCase__				,      )
						return cls.from_config(lowerCamelCase__				,      return_unused_kwargs=lowerCamelCase__				,      **lowerCamelCase__						)





					def    _UpperCAmelCase    (				self				,      lowerCamelCase__				,      lowerCamelCase__ = False				,      **lowerCamelCase__						):
						self.save_config(save_directory=lowerCamelCase__				,      push_to_hub=lowerCamelCase__				,      **lowerCamelCase__						)





					@property
					def    _UpperCAmelCase    (				self						):
						return self._get_compatibles()





					@classmethod
					def    _UpperCAmelCase    (				cls						):
						UpperCAmelCase__:	List[str]									= list(set([cls.__name__] + cls._compatibles						)						)
						UpperCAmelCase__:	Union[str, Any]									= importlib.import_module(__name__.split("."						)[0]						)
						UpperCAmelCase__:	int									= [
						    getattr(lowerCamelCase__				,      lowerCamelCase__						) for c in compatible_classes_str if hasattr(lowerCamelCase__				,      lowerCamelCase__						)
						]
						return compatible_classes