


'''simple docstring'''







import multiprocessing
import os
from typing import BinaryIO, Optional, Union

import fsspec

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.json.json import Json
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader
class 						UpperCamelCase__       (			SCREAMING_SNAKE_CASE_						):



     """simple docstring"""






     def __init__(			self   ,						snake_case__   ,						snake_case__ = None   ,						snake_case__ = None   ,						snake_case__ = None   ,						snake_case__ = False   ,						snake_case__ = False   ,						snake_case__ = None   ,						snake_case__ = None   ,						**snake_case__   ,						):

            '''simple docstring'''

            super().__init__(
                snake_case__   ,						split=snake_case__   ,						features=snake_case__   ,						cache_dir=snake_case__   ,						keep_in_memory=snake_case__   ,						streaming=snake_case__   ,						num_proc=snake_case__   ,						**snake_case__   ,						)
            _lowerCAmelCase     :				Optional[int]       =							field
            _lowerCAmelCase     :				int       =							path_or_paths if isinstance(snake_case__   ,						snake_case__ ) else {self.split: path_or_paths}
            _lowerCAmelCase     :				Any       =							Json(
                cache_dir=snake_case__   ,						data_files=snake_case__   ,						features=snake_case__   ,						field=snake_case__   ,						**snake_case__   ,						)







     def a						(			self ):

            '''simple docstring'''

            if self.streaming:
                   _lowerCAmelCase     :				List[Any]       =							self.builder.as_streaming_dataset(split=self.split )
            # Build regular (map-style) dataset
            else:
                   _lowerCAmelCase     :				Optional[Any]       =							None
                   _lowerCAmelCase     :				Dict       =							None
                   _lowerCAmelCase     :				Dict       =							None
                   _lowerCAmelCase     :				Dict       =							None

                   self.builder.download_and_prepare(
                       download_config=snake_case__   ,						download_mode=snake_case__   ,						verification_mode=snake_case__   ,						base_path=snake_case__   ,						num_proc=self.num_proc   ,						)
                   _lowerCAmelCase     :				str       =							self.builder.as_dataset(
                       split=self.split   ,						verification_mode=snake_case__   ,						in_memory=self.keep_in_memory )
            return dataset
class 						UpperCamelCase__       :



     """simple docstring"""






     def __init__(			self   ,						snake_case__   ,						snake_case__   ,						snake_case__ = None   ,						snake_case__ = None   ,						**snake_case__   ,						):

            '''simple docstring'''

            if num_proc is not None and num_proc <= 0:
                   raise ValueError(F'num_proc {num_proc} must be an integer > 0.' )

            _lowerCAmelCase     :				Any       =							dataset
            _lowerCAmelCase     :				List[str]       =							path_or_buf
            _lowerCAmelCase     :				str       =							batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
            _lowerCAmelCase     :				Optional[int]       =							num_proc
            _lowerCAmelCase     :				Union[str, Any]       =							'utf-8'
            _lowerCAmelCase     :				List[str]       =							to_json_kwargs







     def a						(			self ):

            '''simple docstring'''

            _lowerCAmelCase     :				int       =							self.to_json_kwargs.pop('path_or_buf'   ,						snake_case__ )
            _lowerCAmelCase     :				int       =							self.to_json_kwargs.pop('orient'   ,						'records' )
            _lowerCAmelCase     :				Tuple       =							self.to_json_kwargs.pop('lines'   ,						True if orient == 'records' else False )
            _lowerCAmelCase     :				int       =							self.to_json_kwargs.pop('index'   ,						False if orient in ['split', 'table'] else True )
            _lowerCAmelCase     :				int       =							self.to_json_kwargs.pop('compression'   ,						snake_case__ )

            if compression not in [None, "infer", "gzip", "bz2", "xz"]:
                   raise NotImplementedError(F'`datasets` currently does not support {compression} compression' )

            if isinstance(self.path_or_buf   ,						(str, bytes, os.PathLike) ):
                   with fsspec.open(self.path_or_buf   ,						'wb'   ,						compression=snake_case__ ) as buffer:
                          _lowerCAmelCase     :				Union[str, Any]       =							self._write(file_obj=snake_case__   ,						orient=snake_case__   ,						lines=snake_case__   ,						index=snake_case__   ,						**self.to_json_kwargs )
            else:
                   if compression:
                          raise NotImplementedError(
                              F'The compression parameter is not supported when writing to a buffer, but compression={compression}'
                              ' was passed. Please provide a local path instead.' )
                   _lowerCAmelCase     :				Dict       =							self._write(
                       file_obj=self.path_or_buf   ,						orient=snake_case__   ,						lines=snake_case__   ,						index=snake_case__   ,						**self.to_json_kwargs )
            return written







     def a						(			self   ,						snake_case__ ):

            '''simple docstring'''

            _lowerCAmelCase      ,  _lowerCAmelCase      ,  _lowerCAmelCase      ,  _lowerCAmelCase      ,  _lowerCAmelCase     :				List[str]       =							args

            _lowerCAmelCase     :				Dict       =							query_table(
                table=self.dataset.data   ,						key=slice(snake_case__   ,						offset + self.batch_size )   ,						indices=self.dataset._indices   ,						)
            _lowerCAmelCase     :				Dict       =							batch.to_pandas().to_json(
                path_or_buf=snake_case__   ,						orient=snake_case__   ,						lines=snake_case__   ,						index=snake_case__   ,						**snake_case__ )
            if not json_str.endswith('\n' ):
                   json_str += "\n"
            return json_str.encode(self.encoding )







     def a						(			self   ,						snake_case__   ,						snake_case__   ,						snake_case__   ,						snake_case__   ,						**snake_case__   ,						):

            '''simple docstring'''

            _lowerCAmelCase     :				Tuple       =							0

            if self.num_proc is None or self.num_proc == 1:
                   for offset in logging.tqdm(
                       range(0   ,						len(self.dataset )   ,						self.batch_size )   ,						unit='ba'   ,						disable=not logging.is_progress_bar_enabled()   ,						desc='Creating json from Arrow format'   ,						):
                          _lowerCAmelCase     :				Optional[Any]       =							self._batch_json((offset, orient, lines, index, to_json_kwargs) )
                          written += file_obj.write(snake_case__ )
            else:
                   _lowerCAmelCase      ,  _lowerCAmelCase     :				Union[str, Any]       =							len(self.dataset ), self.batch_size
                   with multiprocessing.Pool(self.num_proc ) as pool:
                          for json_str in logging.tqdm(
                              pool.imap(
                                  self._batch_json   ,						[(offset, orient, lines, index, to_json_kwargs) for offset in range(0   ,						snake_case__   ,						snake_case__ )]   ,						)   ,						total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size   ,						unit='ba'   ,						disable=not logging.is_progress_bar_enabled()   ,						desc='Creating json from Arrow format'   ,						):
                                 written += file_obj.write(snake_case__ )

            return written





'''simple docstring'''







from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_tokenizers_available, is_torch_available


lowerCAmelCase     :  Any											=    {
    """configuration_canine""": ["""CANINE_PRETRAINED_CONFIG_ARCHIVE_MAP""", """CanineConfig"""],
    """tokenization_canine""": ["""CanineTokenizer"""],
}

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       lowerCAmelCase     :  Tuple											=    [
           """CANINE_PRETRAINED_MODEL_ARCHIVE_LIST""",
           """CanineForMultipleChoice""",
           """CanineForQuestionAnswering""",
           """CanineForSequenceClassification""",
           """CanineForTokenClassification""",
           """CanineLayer""",
           """CanineModel""",
           """CaninePreTrainedModel""",
           """load_tf_weights_in_canine""",
       ]


if TYPE_CHECKING:
       from .configuration_canine import CANINE_PRETRAINED_CONFIG_ARCHIVE_MAP, CanineConfig
       from .tokenization_canine import CanineTokenizer

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_canine import (
                  CANINE_PRETRAINED_MODEL_ARCHIVE_LIST,
                  CanineForMultipleChoice,
                  CanineForQuestionAnswering,
                  CanineForSequenceClassification,
                  CanineForTokenClassification,
                  CanineLayer,
                  CanineModel,
                  CaninePreTrainedModel,
                  load_tf_weights_in_canine,
              )


else:
       import sys

       lowerCAmelCase     :  Optional[Any]											=    _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)

