




import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


a_   		=     datasets.utils.logging.get_logger(__name__)

@dataclass
class  _lowerCamelCase						(			datasets.BuilderConfig   ):

      """simple docstring"""

      lowerCAmelCase__			: Optional[datasets.Features]    =					None
      lowerCAmelCase__			: str    =					"utf-8"
      lowerCAmelCase__			: Optional[str]    =					None
      lowerCAmelCase__			: Optional[str]    =					None
      lowerCAmelCase__			: bool    =					True  # deprecated
      lowerCAmelCase__			: Optional[int]    =					None  # deprecated
      lowerCAmelCase__			: int    =					10 << 20  # 10MB
      lowerCAmelCase__			: Optional[bool]    =					None







class  _lowerCamelCase						(			datasets.ArrowBasedBuilder   ):

      """simple docstring"""

      lowerCAmelCase__			: str    =					JsonConfig







      def       snake_case	(     self					: Tuple				):
          if self.config.block_size is not None:
              logger.warning('''The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead'''				)
              __UpperCamelCase  		=						self.config.block_size
          if self.config.use_threads is not True:
              logger.warning(
                  '''The JSON loader parameter `use_threads` is deprecated and doesn\'t have any effect anymore.'''				)
          if self.config.newlines_in_values is not None:
              raise ValueError('''The JSON loader parameter `newlines_in_values` is no longer supported'''				)
          return datasets.DatasetInfo(features=self.config.features				)







      def       snake_case	(     self					: str ,       snake_case					: Optional[int]				):
          if not self.config.data_files:
              raise ValueError(F"At least one data file must be specified, but got data_files={self.config.data_files}"				)
          __UpperCamelCase  		=						dl_manager.download_and_extract(self.config.data_files				)
          if isinstance(snake_case ,       (str, list, tuple)				):
              __UpperCamelCase  		=						data_files
              if isinstance(snake_case ,       snake_case				):
                  __UpperCamelCase  		=						[files]
              __UpperCamelCase  		=						[dl_manager.iter_files(snake_case				) for file in files]
              return [datasets.SplitGenerator(name=datasets.Split.TRAIN ,       gen_kwargs={'''files''': files}				)]
          __UpperCamelCase  		=						[]
          for split_name, files in data_files.items():
              if isinstance(snake_case ,       snake_case				):
                  __UpperCamelCase  		=						[files]
              __UpperCamelCase  		=						[dl_manager.iter_files(snake_case				) for file in files]
              splits.append(datasets.SplitGenerator(name=snake_case ,       gen_kwargs={'''files''': files}				)				)
          return splits







      def       snake_case	(     self					: int ,       snake_case					: pa.Table				):
          if self.config.features is not None:
              # adding missing columns
              for column_name in set(self.config.features				) - set(pa_table.column_names				):
                  __UpperCamelCase  		=						self.config.features.arrow_schema.field(snake_case				).type
                  __UpperCamelCase  		=						pa_table.append_column(snake_case ,       pa.array([None] * len(snake_case				) ,       type=snake_case				)				)
              # more expensive cast to support nested structures with keys in a different order
              # allows str <-> int/float or str to Audio for example
              __UpperCamelCase  		=						table_cast(snake_case ,       self.config.features.arrow_schema				)
          return pa_table







      def       snake_case	(     self					: Optional[int] ,       snake_case					: List[str]				):
          for file_idx, file in enumerate(itertools.chain.from_iterable(snake_case				)				):
              # If the file is one json object and if we need to look at the list of items in one specific field
              if self.config.field is not None:
                  with open(snake_case ,       encoding=self.config.encoding ,       errors=self.config.encoding_errors				) as f:
                      __UpperCamelCase  		=						json.load(snake_case				)

                  # We keep only the field we are interested in
                  __UpperCamelCase  		=						dataset[self.config.field]

                  # We accept two format: a list of dicts or a dict of lists
                  if isinstance(snake_case ,       (list, tuple)				):
                      __UpperCamelCase  		=						set().union(*[row.keys() for row in dataset]				)
                      __UpperCamelCase  		=						{col: [row.get(snake_case				) for row in dataset] for col in keys}
                  else:
                      __UpperCamelCase  		=						dataset
                  __UpperCamelCase  		=						pa.Table.from_pydict(snake_case				)
                  yield file_idx, self._cast_table(snake_case				)

              # If the file has one json object per line
              else:
                  with open(snake_case ,       '''rb'''				) as f:
                      __UpperCamelCase  		=						0
                      # Use block_size equal to the chunk size divided by 32 to leverage multithreading
                      # Set a default minimum value of 16kB if the chunk size is really small
                      __UpperCamelCase  		=						max(self.config.chunksize // 32 ,       16 << 10				)
                      __UpperCamelCase  		=						(
                          self.config.encoding_errors if self.config.encoding_errors is not None else '''strict'''
                      )
                      while True:
                          __UpperCamelCase  		=						f.read(self.config.chunksize				)
                          if not batch:
                              break
                          # Finish current line
                          try:
                              batch += f.readline()
                          except (AttributeError, io.UnsupportedOperation):
                              batch += readline(snake_case				)
                          # PyArrow only accepts utf-8 encoded bytes
                          if self.config.encoding != "utf-8":
                              __UpperCamelCase  		=						batch.decode(self.config.encoding ,       errors=snake_case				).encode('''utf-8'''				)
                          try:
                              while True:
                                  try:
                                      __UpperCamelCase  		=						paj.read_json(
                                          io.BytesIO(snake_case				) ,       read_options=paj.ReadOptions(block_size=snake_case				)				)
                                      break
                                  except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
                                      if (
                                          isinstance(snake_case ,       pa.ArrowInvalid				)
                                          and "straddling" not in str(snake_case				)
                                          or block_size > len(snake_case				)
                                      ):
                                          raise
                                      else:
                                          # Increase the block size in case it was too small.
                                          # The block size will be reset for the next file.
                                          logger.debug(
                                              F"Batch of {len(snake_case				)} bytes couldn't be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}."				)
                                          block_size *= 2
                          except pa.ArrowInvalid as e:
                              try:
                                  with open(
                                      snake_case ,       encoding=self.config.encoding ,       errors=self.config.encoding_errors				) as f:
                                      __UpperCamelCase  		=						json.load(snake_case				)
                              except json.JSONDecodeError:
                                  logger.error(F"Failed to read file '{file}' with error {type(snake_case				)}: {e}"				)
                                  raise e
                              # If possible, parse the file as a list of json objects and exit the loop
                              if isinstance(snake_case ,       snake_case				):  # list is the only sequence type supported in JSON
                                  try:
                                      __UpperCamelCase  		=						set().union(*[row.keys() for row in dataset]				)
                                      __UpperCamelCase  		=						{col: [row.get(snake_case				) for row in dataset] for col in keys}
                                      __UpperCamelCase  		=						pa.Table.from_pydict(snake_case				)
                                  except (pa.ArrowInvalid, AttributeError) as e:
                                      logger.error(F"Failed to read file '{file}' with error {type(snake_case				)}: {e}"				)
                                      raise ValueError(F"Not able to read records in the JSON file at {file}."				) from None
                                  yield file_idx, self._cast_table(snake_case				)
                                  break
                              else:
                                  logger.error(F"Failed to read file '{file}' with error {type(snake_case				)}: {e}"				)
                                  raise ValueError(
                                      F"Not able to read records in the JSON file at {file}. "
                                      F"You should probably indicate the field of the JSON file containing your records. "
                                      F"This JSON file contain the following fields: {str(list(dataset.keys()				)				)}. "
                                      F"Select the correct one and provide it as `field='XXX'` to the dataset loading method. "				) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
                          yield (file_idx, batch_idx), self._cast_table(snake_case				)
                          batch_idx += 1












from ...configuration_utils import PretrainedConfig
from ...utils import logging


a_   		=     logging.get_logger(__name__)

a_   		=     {
    "RWKV/rwkv-4-169m-pile": "https://huggingface.co/RWKV/rwkv-4-169m-pile/resolve/main/config.json",
    "RWKV/rwkv-4-430m-pile": "https://huggingface.co/RWKV/rwkv-4-430m-pile/resolve/main/config.json",
    "RWKV/rwkv-4-1b5-pile": "https://huggingface.co/RWKV/rwkv-4-1b5-pile/resolve/main/config.json",
    "RWKV/rwkv-4-3b-pile": "https://huggingface.co/RWKV/rwkv-4-3b-pile/resolve/main/config.json",
    "RWKV/rwkv-4-7b-pile": "https://huggingface.co/RWKV/rwkv-4-7b-pile/resolve/main/config.json",
    "RWKV/rwkv-4-14b-pile": "https://huggingface.co/RWKV/rwkv-4-14b-pile/resolve/main/config.json",
    "RWKV/rwkv-raven-1b5": "https://huggingface.co/RWKV/rwkv-raven-1b5/resolve/main/config.json",
    "RWKV/rwkv-raven-3b": "https://huggingface.co/RWKV/rwkv-raven-3b/resolve/main/config.json",
    "RWKV/rwkv-raven-7b": "https://huggingface.co/RWKV/rwkv-raven-7b/resolve/main/config.json",
    "RWKV/rwkv-raven-14b": "https://huggingface.co/RWKV/rwkv-raven-14b/resolve/main/config.json",
}

class  _lowerCamelCase						(			_SCREAMING_SNAKE_CASE   ):

      """simple docstring"""

      lowerCAmelCase__			: Union[str, Any]    =					"rwkv"
      lowerCAmelCase__			: Union[str, Any]    =					{"max_position_embeddings": "context_length"}







      def __init__(     self					: Optional[int] ,       snake_case					: Optional[Any]=50277 ,       snake_case					: str=1024 ,       snake_case					: str=4096 ,       snake_case					: Optional[Any]=32 ,       snake_case					: Union[str, Any]=None ,       snake_case					: Optional[Any]=None ,       snake_case					: Optional[Any]=1E-5 ,       snake_case					: List[str]=0 ,       snake_case					: Optional[Any]=0 ,       snake_case					: Union[str, Any]=6 ,       snake_case					: Tuple=False ,       snake_case					: Any=True ,       **snake_case					: List[str] ,       ):
          __UpperCamelCase  		=						vocab_size
          __UpperCamelCase  		=						context_length
          __UpperCamelCase  		=						hidden_size
          __UpperCamelCase  		=						num_hidden_layers
          __UpperCamelCase  		=						attention_hidden_size if attention_hidden_size is not None else hidden_size
          __UpperCamelCase  		=						intermediate_size if intermediate_size is not None else 4 * hidden_size
          __UpperCamelCase  		=						layer_norm_epsilon
          __UpperCamelCase  		=						rescale_every
          __UpperCamelCase  		=						use_cache

          __UpperCamelCase  		=						bos_token_id
          __UpperCamelCase  		=						eos_token_id

          super().__init__(
              tie_word_embeddings=snake_case ,       bos_token_id=snake_case ,       eos_token_id=snake_case ,       **snake_case				)






