


'''simple docstring'''



import multiprocessing
import os
from typing import BinaryIO, Optional, Union

import fsspec

from .. import Dataset, Features, NamedSplit, config
from ..formatting import query_table
from ..packaged_modules.json.json import Json
from ..utils import logging
from ..utils.typing import NestedDataStructureLike, PathLike
from .abc import AbstractDatasetReader





class   UpperCAmelCase_    (_UpperCAmelCase    ):





    """simple docstring"""






    def __init__(				self     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_ = None     ,	SCREAMING_SNAKE_CASE_ = None     ,	SCREAMING_SNAKE_CASE_ = None     ,	SCREAMING_SNAKE_CASE_ = False     ,	SCREAMING_SNAKE_CASE_ = False     ,	SCREAMING_SNAKE_CASE_ = None     ,	SCREAMING_SNAKE_CASE_ = None     ,	**SCREAMING_SNAKE_CASE_     ,	)							->	Dict:
         super().__init__(
             SCREAMING_SNAKE_CASE_     ,	split=SCREAMING_SNAKE_CASE_     ,	features=SCREAMING_SNAKE_CASE_     ,	cache_dir=SCREAMING_SNAKE_CASE_     ,	keep_in_memory=SCREAMING_SNAKE_CASE_     ,	streaming=SCREAMING_SNAKE_CASE_     ,	num_proc=SCREAMING_SNAKE_CASE_     ,	**SCREAMING_SNAKE_CASE_     ,	)
         __lowerCamelCase  :      Any														=						field
         __lowerCamelCase  :      Tuple														=						path_or_paths if isinstance(SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_      ) else {self.split: path_or_paths}
         __lowerCamelCase  :      int														=						Json(
             cache_dir=SCREAMING_SNAKE_CASE_     ,	data_files=SCREAMING_SNAKE_CASE_     ,	features=SCREAMING_SNAKE_CASE_     ,	field=SCREAMING_SNAKE_CASE_     ,	**SCREAMING_SNAKE_CASE_     ,	)







    def  lowercase_							(				self      )							->	List[str]:
         # Build iterable dataset
         if self.streaming:
              __lowerCamelCase  :      str														=						self.builder.as_streaming_dataset(split=self.split      )
         # Build regular (map-style) dataset
         else:
              __lowerCamelCase  :      List[Any]														=						None
              __lowerCamelCase  :      Optional[Any]														=						None
              __lowerCamelCase  :      Union[str, Any]														=						None
              __lowerCamelCase  :      Dict														=						None

              self.builder.download_and_prepare(
                  download_config=SCREAMING_SNAKE_CASE_     ,	download_mode=SCREAMING_SNAKE_CASE_     ,	verification_mode=SCREAMING_SNAKE_CASE_     ,	base_path=SCREAMING_SNAKE_CASE_     ,	num_proc=self.num_proc     ,	)
              __lowerCamelCase  :      List[str]														=						self.builder.as_dataset(
                  split=self.split     ,	verification_mode=SCREAMING_SNAKE_CASE_     ,	in_memory=self.keep_in_memory      )
         return dataset







class   UpperCAmelCase_    :





    """simple docstring"""






    def __init__(				self     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_ = None     ,	SCREAMING_SNAKE_CASE_ = None     ,	**SCREAMING_SNAKE_CASE_     ,	)							->	int:
         if num_proc is not None and num_proc <= 0:
              raise ValueError(f'num_proc {num_proc} must be an integer > 0.'      )

         __lowerCamelCase  :      Tuple														=						dataset
         __lowerCamelCase  :      Dict														=						path_or_buf
         __lowerCamelCase  :      int														=						batch_size if batch_size else config.DEFAULT_MAX_BATCH_SIZE
         __lowerCamelCase  :      str														=						num_proc
         __lowerCamelCase  :      Union[str, Any]														=						'utf-8'
         __lowerCamelCase  :      str														=						to_json_kwargs







    def  lowercase_							(				self      )							->	int:
         __lowerCamelCase  :      Optional[Any]														=						self.to_json_kwargs.pop('path_or_buf'     ,	SCREAMING_SNAKE_CASE_      )
         __lowerCamelCase  :      Tuple														=						self.to_json_kwargs.pop('orient'     ,	'records'      )
         __lowerCamelCase  :      List[Any]														=						self.to_json_kwargs.pop('lines'     ,	True if orient == 'records' else False      )
         __lowerCamelCase  :      List[str]														=						self.to_json_kwargs.pop('index'     ,	False if orient in ['split', 'table'] else True      )
         __lowerCamelCase  :      Tuple														=						self.to_json_kwargs.pop('compression'     ,	SCREAMING_SNAKE_CASE_      )

         if compression not in [None, "infer", "gzip", "bz2", "xz"]:
              raise NotImplementedError(f'`datasets` currently does not support {compression} compression'      )

         if isinstance(self.path_or_buf     ,	(str, bytes, os.PathLike)      ):
              with fsspec.open(self.path_or_buf     ,	'wb'     ,	compression=SCREAMING_SNAKE_CASE_      ) as buffer:
                   __lowerCamelCase  :      List[Any]														=						self._write(file_obj=SCREAMING_SNAKE_CASE_     ,	orient=SCREAMING_SNAKE_CASE_     ,	lines=SCREAMING_SNAKE_CASE_     ,	index=SCREAMING_SNAKE_CASE_     ,	**self.to_json_kwargs      )
         else:
              if compression:
                   raise NotImplementedError(
                       f'The compression parameter is not supported when writing to a buffer, but compression={compression}'
                       ' was passed. Please provide a local path instead.'      )
              __lowerCamelCase  :      Dict														=						self._write(
                  file_obj=self.path_or_buf     ,	orient=SCREAMING_SNAKE_CASE_     ,	lines=SCREAMING_SNAKE_CASE_     ,	index=SCREAMING_SNAKE_CASE_     ,	**self.to_json_kwargs      )
         return written







    def  lowercase_							(				self     ,	SCREAMING_SNAKE_CASE_      )							->	List[str]:
         __lowerCamelCase							,			__lowerCamelCase							,			__lowerCamelCase							,			__lowerCamelCase							,			__lowerCamelCase  :      str														=						args

         __lowerCamelCase  :      List[str]														=						query_table(
             table=self.dataset.data     ,	key=slice(SCREAMING_SNAKE_CASE_     ,	offset + self.batch_size      )     ,	indices=self.dataset._indices     ,	)
         __lowerCamelCase  :      Tuple														=						batch.to_pandas().to_json(
             path_or_buf=SCREAMING_SNAKE_CASE_     ,	orient=SCREAMING_SNAKE_CASE_     ,	lines=SCREAMING_SNAKE_CASE_     ,	index=SCREAMING_SNAKE_CASE_     ,	**SCREAMING_SNAKE_CASE_      )
         if not json_str.endswith('\n'      ):
              json_str += "\n"
         return json_str.encode(self.encoding      )







    def  lowercase_							(				self     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_     ,	**SCREAMING_SNAKE_CASE_     ,	)							->	int:
         __lowerCamelCase  :      Any														=						0

         if self.num_proc is None or self.num_proc == 1:
              for offset in logging.tqdm(
                  range(0     ,	len(self.dataset      )     ,	self.batch_size      )     ,	unit='ba'     ,	disable=not logging.is_progress_bar_enabled()     ,	desc='Creating json from Arrow format'     ,	):
                   __lowerCamelCase  :      Optional[int]														=						self._batch_json((offset, orient, lines, index, to_json_kwargs)      )
                   written += file_obj.write(SCREAMING_SNAKE_CASE_      )
         else:
              __lowerCamelCase							,			__lowerCamelCase  :      Optional[Any]														=						len(self.dataset      ), self.batch_size
              with multiprocessing.Pool(self.num_proc      ) as pool:
                   for json_str in logging.tqdm(
                       pool.imap(
                           self._batch_json     ,	[(offset, orient, lines, index, to_json_kwargs) for offset in range(0     ,	SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_      )]     ,	)     ,	total=(num_rows // batch_size) + 1 if num_rows % batch_size else num_rows // batch_size     ,	unit='ba'     ,	disable=not logging.is_progress_bar_enabled()     ,	desc='Creating json from Arrow format'     ,	):
                        written += file_obj.write(SCREAMING_SNAKE_CASE_      )

         return written










'''simple docstring'''



from collections.abc import Generator
from math import sin



def      UpperCAmelCase__   (      UpperCAmelCase_       :							bytes				)	->						bytes:
     if len(UpperCAmelCase_				) != 32:
          raise ValueError('Input must be of length 32'				)

     __lowerCamelCase  :      Dict														=						B''
     for i in [3, 2, 1, 0]:
          little_endian += string_aa[8 * i : 8 * i + 8]
     return little_endian



def      UpperCAmelCase__   (      UpperCAmelCase_       :							int				)	->						bytes:
     if i < 0:
          raise ValueError('Input must be non-negative'				)

     __lowerCamelCase  :      Union[str, Any]														=						format(UpperCAmelCase_ ,       '08x'				)[-8:]
     __lowerCamelCase  :      str														=						B''
     for i in [3, 2, 1, 0]:
          little_endian_hex += hex_rep[2 * i : 2 * i + 2].encode('utf-8'				)
     return little_endian_hex



def      UpperCAmelCase__   (      UpperCAmelCase_       :							bytes				)	->						bytes:
     __lowerCamelCase  :      Optional[Any]														=						B''
     for char in message:
          bit_string += format(UpperCAmelCase_ ,       '08b'				).encode('utf-8'				)
     __lowerCamelCase  :      List[str]														=						format(len(UpperCAmelCase_				) ,       '064b'				).encode('utf-8'				)

     # Pad bit_string to a multiple of 512 chars
     bit_string += b"1"
     while len(UpperCAmelCase_				) % 5_12 != 4_48:
          bit_string += b"0"
     bit_string += to_little_endian(start_len[32:]				) + to_little_endian(start_len[:32]				)

     return bit_string



def      UpperCAmelCase__   (      UpperCAmelCase_       :							bytes				)	->						Generator[list[int], None, None]:
     if len(UpperCAmelCase_				) % 5_12 != 0:
          raise ValueError('Input must have length that\'s a multiple of 512'				)

     for pos in range(0 ,       len(UpperCAmelCase_				) ,       5_12				):
          __lowerCamelCase  :      Any														=						bit_string[pos : pos + 5_12]
          __lowerCamelCase  :      Optional[int]														=						[]
          for i in range(0 ,       5_12 ,       32				):
               block_words.append(int(to_little_endian(block[i : i + 32]				) ,       2				)				)
          yield block_words



def      UpperCAmelCase__   (      UpperCAmelCase_       :							int				)	->						int:
     if i < 0:
          raise ValueError('Input must be non-negative'				)

     __lowerCamelCase  :      List[Any]														=						format(UpperCAmelCase_ ,       '032b'				)
     __lowerCamelCase  :      Optional[int]														=						''
     for c in i_str:
          new_str += "1" if c == "0" else "0"
     return int(UpperCAmelCase_ ,       2				)



def      UpperCAmelCase__   (      UpperCAmelCase_       :							int ,       UpperCAmelCase_       :							int				)	->						int:
     return (a + b) % 2**32



def      UpperCAmelCase__   (      UpperCAmelCase_       :							int ,       UpperCAmelCase_       :							int				)	->						int:
     if i < 0:
          raise ValueError('Input must be non-negative'				)
     if shift < 0:
          raise ValueError('Shift must be non-negative'				)
     return ((i << shift) ^ (i >> (32 - shift))) % 2**32



def      UpperCAmelCase__   (      UpperCAmelCase_       :							bytes				)	->						bytes:
     __lowerCamelCase  :      Optional[Any]														=						preprocess(UpperCAmelCase_				)

     __lowerCamelCase  :      Union[str, Any]														=						[int(2**32 * abs(sin(i + 1				)				)				) for i in range(64				)]

     # Starting states
     __lowerCamelCase  :      Dict														=						0x67_45_23_01
     __lowerCamelCase  :      Union[str, Any]														=						0xef_cd_ab_89
     __lowerCamelCase  :      Optional[Any]														=						0x98_ba_dc_fe
     __lowerCamelCase  :      Union[str, Any]														=						0x10_32_54_76

     __lowerCamelCase  :      List[str]														=						[
         7,
         12,
         17,
         22,
         7,
         12,
         17,
         22,
         7,
         12,
         17,
         22,
         7,
         12,
         17,
         22,
         5,
         9,
         14,
         20,
         5,
         9,
         14,
         20,
         5,
         9,
         14,
         20,
         5,
         9,
         14,
         20,
         4,
         11,
         16,
         23,
         4,
         11,
         16,
         23,
         4,
         11,
         16,
         23,
         4,
         11,
         16,
         23,
         6,
         10,
         15,
         21,
         6,
         10,
         15,
         21,
         6,
         10,
         15,
         21,
         6,
         10,
         15,
         21,
     ]

     # Process bit string in chunks, each with 16 32-char words
     for block_words in get_block_words(UpperCAmelCase_				):
          __lowerCamelCase  :      Dict														=						aa
          __lowerCamelCase  :      Tuple														=						ba
          __lowerCamelCase  :      List[Any]														=						ca
          __lowerCamelCase  :      Dict														=						da

          # Hash current chunk
          for i in range(64				):
               if i <= 15:
                    # f = (b & c) | (not_32(b) & d)     # Alternate definition for f
                    __lowerCamelCase  :      List[str]														=						d ^ (b & (c ^ d))
                    __lowerCamelCase  :      Optional[int]														=						i
               elif i <= 31:
                    # f = (d & b) | (not_32(d) & c)     # Alternate definition for f
                    __lowerCamelCase  :      Optional[int]														=						c ^ (d & (b ^ c))
                    __lowerCamelCase  :      Tuple														=						(5 * i + 1) % 16
               elif i <= 47:
                    __lowerCamelCase  :      str														=						b ^ c ^ d
                    __lowerCamelCase  :      Any														=						(3 * i + 5) % 16
               else:
                    __lowerCamelCase  :      Union[str, Any]														=						c ^ (b | not_aa(UpperCAmelCase_				))
                    __lowerCamelCase  :      int														=						(7 * i) % 16
               __lowerCamelCase  :      Optional[int]														=						(f + a + added_consts[i] + block_words[g]) % 2**32
               __lowerCamelCase  :      Optional[Any]														=						d
               __lowerCamelCase  :      Tuple														=						c
               __lowerCamelCase  :      Optional[int]														=						b
               __lowerCamelCase  :      List[str]														=						sum_aa(UpperCAmelCase_ ,       left_rotate_aa(UpperCAmelCase_ ,       shift_amounts[i]				)				)

          # Add hashed chunk to running total
          __lowerCamelCase  :      int														=						sum_aa(UpperCAmelCase_ ,       UpperCAmelCase_				)
          __lowerCamelCase  :      str														=						sum_aa(UpperCAmelCase_ ,       UpperCAmelCase_				)
          __lowerCamelCase  :      List[Any]														=						sum_aa(UpperCAmelCase_ ,       UpperCAmelCase_				)
          __lowerCamelCase  :      List[str]														=						sum_aa(UpperCAmelCase_ ,       UpperCAmelCase_				)

     __lowerCamelCase  :      Dict														=						reformat_hex(UpperCAmelCase_				) + reformat_hex(UpperCAmelCase_				) + reformat_hex(UpperCAmelCase_				) + reformat_hex(UpperCAmelCase_				)
     return digest


if __name__ == "__main__":
      import doctest

      doctest.testmod()






