
import io
import itertools
import json
from dataclasses import dataclass
from typing import Optional

import pyarrow as pa
import pyarrow.json as paj

import datasets
from datasets.table import table_cast
from datasets.utils.file_utils import readline


UpperCamelCase_						:     Union[str, Any]	       =							datasets.utils.logging.get_logger(__name__)




@dataclass
class   __lowercase						(						datasets.BuilderConfig       ):
	_A       =   None
	_A       =   "utf-8"
	_A       =   None
	_A       =   None
	_A       =   True  # deprecated
	_A       =   None  # deprecated
	_A       =   10 << 20  # 10MB
	_A       =   None




class   __lowercase						(						datasets.ArrowBasedBuilder       ):
	_A       =   JsonConfig






	def       _a(self				: List[Any]    ) ->			Dict:
							if self.config.block_size is not None:
													logger.warning("The JSON loader parameter `block_size` is deprecated. Please use `chunksize` instead"    )
													_lowercase       :	Union[str, Any]           =   self.config.block_size
							if self.config.use_threads is not True:
													logger.warning(
													    "The JSON loader parameter `use_threads` is deprecated and doesn't have any effect anymore."    )
							if self.config.newlines_in_values is not None:
													raise ValueError("The JSON loader parameter `newlines_in_values` is no longer supported"    )
							return datasets.DatasetInfo(features=self.config.features    )






	def       _a(self				: Any      ,	snake_case				: Union[str, Any]    ) ->			List[str]:
							if not self.config.data_files:
													raise ValueError(F"""At least one data file must be specified, but got data_files={self.config.data_files}"""    )
							_lowercase       :	str           =   dl_manager.download_and_extract(self.config.data_files    )
							if isinstance(snake_case      ,	(str, list, tuple)    ):
													_lowercase       :	int           =   data_files
													if isinstance(snake_case      ,	snake_case    ):
																			_lowercase       :	Union[str, Any]           =   [files]
													_lowercase       :	Union[str, Any]           =   [dl_manager.iter_files(snake_case    ) for file in files]
													return [datasets.SplitGenerator(name=datasets.Split.TRAIN      ,	gen_kwargs={"files": files}    )]
							_lowercase       :	Union[str, Any]           =   []
							for split_name, files in data_files.items():
													if isinstance(snake_case      ,	snake_case    ):
																			_lowercase       :	List[Any]           =   [files]
													_lowercase       :	Optional[Any]           =   [dl_manager.iter_files(snake_case    ) for file in files]
													splits.append(datasets.SplitGenerator(name=snake_case      ,	gen_kwargs={"files": files}    )    )
							return splits






	def       _a(self				: List[Any]      ,	snake_case				: pa.Table    ) ->			pa.Table:
							if self.config.features is not None:
													# adding missing columns
													for column_name in set(self.config.features    ) - set(pa_table.column_names    ):
																			_lowercase       :	Any           =   self.config.features.arrow_schema.field(snake_case    ).type
																			_lowercase       :	Optional[Any]           =   pa_table.append_column(snake_case      ,	pa.array([None] * len(snake_case    )      ,	type=snake_case    )    )
													# more expensive cast to support nested structures with keys in a different order
													# allows str <-> int/float or str to Audio for example
													_lowercase       :	str           =   table_cast(snake_case      ,	self.config.features.arrow_schema    )
							return pa_table






	def       _a(self				: str      ,	snake_case				: Tuple    ) ->			Tuple:
							for file_idx, file in enumerate(itertools.chain.from_iterable(snake_case    )    ):
													# If the file is one json object and if we need to look at the list of items in one specific field
													if self.config.field is not None:
																			with open(snake_case      ,	encoding=self.config.encoding      ,	errors=self.config.encoding_errors    ) as f:
																									_lowercase       :	Any           =   json.load(snake_case    )

																			# We keep only the field we are interested in
																			_lowercase       :	List[Any]           =   dataset[self.config.field]

																			# We accept two format: a list of dicts or a dict of lists
																			if isinstance(snake_case      ,	(list, tuple)    ):
																									_lowercase       :	Optional[int]           =   set().union(*[row.keys() for row in dataset]    )
																									_lowercase       :	Union[str, Any]           =   {col: [row.get(snake_case    ) for row in dataset] for col in keys}
																			else:
																									_lowercase       :	List[Any]           =   dataset
																			_lowercase       :	int           =   pa.Table.from_pydict(snake_case    )
																			yield file_idx, self._cast_table(snake_case    )

													# If the file has one json object per line
													else:
																			with open(snake_case      ,	"rb"    ) as f:
																									_lowercase       :	Union[str, Any]           =   0
																									# Use block_size equal to the chunk size divided by 32 to leverage multithreading
																									# Set a default minimum value of 16kB if the chunk size is really small
																									_lowercase       :	Dict           =   max(self.config.chunksize // 32      ,	16 << 10    )
																									_lowercase       :	int           =   (
																									    self.config.encoding_errors if self.config.encoding_errors is not None else "strict"
																									)
																									while True:
																															_lowercase       :	Tuple           =   f.read(self.config.chunksize    )
																															if not batch:
																																					break
																															# Finish current line
																															try:
																																					batch += f.readline()
																															except (AttributeError, io.UnsupportedOperation):
																																					batch += readline(snake_case    )
																															# PyArrow only accepts utf-8 encoded bytes
																															if self.config.encoding != "utf-8":
																																					_lowercase       :	Union[str, Any]           =   batch.decode(self.config.encoding      ,	errors=snake_case    ).encode("utf-8"    )
																															try:
																																					while True:
																																											try:
																																																	_lowercase       :	Optional[Any]           =   paj.read_json(
																																																	    io.BytesIO(snake_case    )      ,	read_options=paj.ReadOptions(block_size=snake_case    )    )
																																																	break
																																											except (pa.ArrowInvalid, pa.ArrowNotImplementedError) as e:
																																																	if (
																																																	    isinstance(snake_case      ,	pa.ArrowInvalid    )
																																																	    and "straddling" not in str(snake_case    )
																																																	    or block_size > len(snake_case    )
																																																	):
																																																							raise
																																																	else:
																																																							# Increase the block size in case it was too small.
																																																							# The block size will be reset for the next file.
																																																							logger.debug(
																																																							    F"""Batch of {len(snake_case    )} bytes couldn't be parsed with block_size={block_size}. Retrying with block_size={block_size * 2}."""    )
																																																							block_size *= 2
																															except pa.ArrowInvalid as e:
																																					try:
																																											with open(
																																											    snake_case      ,	encoding=self.config.encoding      ,	errors=self.config.encoding_errors    ) as f:
																																																	_lowercase       :	Any           =   json.load(snake_case    )
																																					except json.JSONDecodeError:
																																											logger.error(F"""Failed to read file '{file}' with error {type(snake_case    )}: {e}"""    )
																																											raise e
																																					# If possible, parse the file as a list of json objects and exit the loop
																																					if isinstance(snake_case      ,	snake_case    ):  # list is the only sequence type supported in JSON
																																											try:
																																																	_lowercase       :	int           =   set().union(*[row.keys() for row in dataset]    )
																																																	_lowercase       :	Dict           =   {col: [row.get(snake_case    ) for row in dataset] for col in keys}
																																																	_lowercase       :	str           =   pa.Table.from_pydict(snake_case    )
																																											except (pa.ArrowInvalid, AttributeError) as e:
																																																	logger.error(F"""Failed to read file '{file}' with error {type(snake_case    )}: {e}"""    )
																																																	raise ValueError(F"""Not able to read records in the JSON file at {file}."""    ) from None
																																											yield file_idx, self._cast_table(snake_case    )
																																											break
																																					else:
																																											logger.error(F"""Failed to read file '{file}' with error {type(snake_case    )}: {e}"""    )
																																											raise ValueError(
																																											    F"""Not able to read records in the JSON file at {file}. """
																																											    F"""You should probably indicate the field of the JSON file containing your records. """
																																											    F"""This JSON file contain the following fields: {str(list(dataset.keys()    )    )}. """
																																											    F"""Select the correct one and provide it as `field='XXX'` to the dataset loading method. """    ) from None
                        # Uncomment for debugging (will print the Arrow table size and elements)
                        # logger.warning(f"pa_table: {pa_table} num rows: {pa_table.num_rows}")
                        # logger.warning('\n'.join(str(pa_table.slice(i, 1).to_pydict()) for i in range(pa_table.num_rows)))
																															yield (file_idx, batch_idx), self._cast_table(snake_case    )
																															batch_idx += 1






import importlib.metadata
import warnings
from copy import deepcopy

from packaging import version

from ..utils import logging
from .import_utils import is_accelerate_available, is_bitsandbytes_available


if is_bitsandbytes_available():
							import bitsandbytes as bnb
							import torch
							import torch.nn as nn

							from ..pytorch_utils import ConvaD

if is_accelerate_available():
							from accelerate import init_empty_weights
							from accelerate.utils import find_tied_parameters

UpperCamelCase_						:     int	       =							logging.get_logger(__name__)



def 		UpperCamelCase     (       _UpperCAmelCase  :   Optional[Any] ,			_UpperCAmelCase  :   Optional[Any] ,			_UpperCAmelCase  :   Optional[Any] ,			_UpperCAmelCase  :   Dict=None ,			_UpperCAmelCase  :   Dict=None  )   ->			Tuple:






						'''simple docstring'''

						if "." in tensor_name:
												_lowercase       :	Dict           =   tensor_name.split("."  )
												for split in splits[:-1]:
																		_lowercase       :	Optional[int]           =   getattr(_UpperCAmelCase ,			_UpperCAmelCase  )
																		if new_module is None:
																								raise ValueError(f"""{module} has no attribute {split}."""  )
																		_lowercase       :	List[Any]           =   new_module
												_lowercase       :	Dict           =   splits[-1]

						if tensor_name not in module._parameters and tensor_name not in module._buffers:
												raise ValueError(f"""{module} does not have a parameter or a buffer named {tensor_name}."""  )
						_lowercase       :	Dict           =   tensor_name in module._buffers
						_lowercase       :	Union[str, Any]           =   getattr(_UpperCAmelCase ,			_UpperCAmelCase  )

						if old_value.device == torch.device("meta"  ) and device not in ["meta", torch.device("meta"  )] and value is None:
												raise ValueError(f"""{tensor_name} is on the meta device, we need a `value` to put in on {device}."""  )

						_lowercase       :	int           =   False
						_lowercase       :	str           =   False
						if is_buffer or not is_bitsandbytes_available():
												_lowercase       :	int           =   False
												_lowercase       :	List[str]           =   False
						else:
												_lowercase       :	List[str]           =   hasattr(bnb.nn ,			"Params4bit"  ) and isinstance(module._parameters[tensor_name] ,			bnb.nn.Paramsabit  )
												_lowercase       :	int           =   isinstance(module._parameters[tensor_name] ,			bnb.nn.IntaParams  )

						if is_abit or is_abit:
												_lowercase       :	Optional[int]           =   module._parameters[tensor_name]
												if param.device.type != "cuda":
																		if value is None:
																								_lowercase       :	int           =   old_value.to(_UpperCAmelCase  )
																		elif isinstance(_UpperCAmelCase ,			torch.Tensor  ):
																								_lowercase       :	Dict           =   value.to("cpu"  )
																								if value.dtype == torch.inta:
																														_lowercase       :	str           =   version.parse(importlib.metadata.version("bitsandbytes"  )  ) > version.parse(
																														    "0.37.2"  )
																														if not is_abit_serializable:
																																				raise ValueError(
																																				    "Detected int8 weights but the version of bitsandbytes is not compatible with int8 serialization. "
																																				    "Make sure to download the latest `bitsandbytes` version. `pip install --upgrade bitsandbytes`."  )
																		else:
																								_lowercase       :	List[str]           =   torch.tensor(_UpperCAmelCase ,			device="cpu"  )

																		# Support models using `Conv1D` in place of `nn.Linear` (e.g. gpt2) by transposing the weight matrix prior to quantization.
																		# Since weights are saved in the correct "orientation", we skip transposing when loading.
																		if issubclass(module.source_cls ,			_UpperCAmelCase  ) and fpaa_statistics is None:
																								_lowercase       :	List[Any]           =   new_value.T

																		_lowercase       :	List[str]           =   old_value.__dict__
																		if is_abit:
																								_lowercase       :	Optional[Any]           =   bnb.nn.IntaParams(_UpperCAmelCase ,			requires_grad=_UpperCAmelCase ,			**_UpperCAmelCase  ).to(_UpperCAmelCase  )
																		elif is_abit:
																								_lowercase       :	Dict           =   bnb.nn.Paramsabit(_UpperCAmelCase ,			requires_grad=_UpperCAmelCase ,			**_UpperCAmelCase  ).to(_UpperCAmelCase  )

																		_lowercase       :	str           =   new_value
																		if fpaa_statistics is not None:
																								setattr(module.weight ,			"SCB" ,			fpaa_statistics.to(_UpperCAmelCase  )  )

						else:
												if value is None:
																		_lowercase       :	Optional[Any]           =   old_value.to(_UpperCAmelCase  )
												elif isinstance(_UpperCAmelCase ,			torch.Tensor  ):
																		_lowercase       :	Tuple           =   value.to(_UpperCAmelCase  )
												else:
																		_lowercase       :	Optional[Any]           =   torch.tensor(_UpperCAmelCase ,			device=_UpperCAmelCase  )

												if is_buffer:
																		_lowercase       :	str           =   new_value
												else:
																		_lowercase       :	int           =   nn.Parameter(_UpperCAmelCase ,			requires_grad=old_value.requires_grad  )
																		_lowercase       :	str           =   new_value



def 		UpperCamelCase     (       _UpperCAmelCase  :   Any ,			_UpperCAmelCase  :   str=None ,			_UpperCAmelCase  :   List[str]=None ,			_UpperCAmelCase  :   Tuple=None ,			_UpperCAmelCase  :   Tuple=False  )   ->			int:






						'''simple docstring'''

						for name, module in model.named_children():
												if current_key_name is None:
																		_lowercase       :	int           =   []
												current_key_name.append(_UpperCAmelCase  )

												if (isinstance(_UpperCAmelCase ,			nn.Linear  ) or isinstance(_UpperCAmelCase ,			_UpperCAmelCase  )) and name not in modules_to_not_convert:
																		# Check if the current key is not in the `modules_to_not_convert`
																		if not any(key in ".".join(_UpperCAmelCase  ) for key in modules_to_not_convert  ):
																								with init_empty_weights():
																														if isinstance(_UpperCAmelCase ,			_UpperCAmelCase  ):
																																				_lowercase      ,    _lowercase       :	Tuple           =   module.weight.shape
																														else:
																																				_lowercase       :	List[Any]           =   module.in_features
																																				_lowercase       :	List[str]           =   module.out_features

																														if quantization_config.quantization_method() == "llm_int8":
																																				_lowercase       :	Union[str, Any]           =   bnb.nn.LinearabitLt(
																																				    _UpperCAmelCase ,			_UpperCAmelCase ,			module.bias is not None ,			has_fpaa_weights=quantization_config.llm_inta_has_fpaa_weight ,			threshold=quantization_config.llm_inta_threshold ,			)
																																				_lowercase       :	Tuple           =   True
																														else:
																																				if (
																																				    quantization_config.llm_inta_skip_modules is not None
																																				    and name in quantization_config.llm_inta_skip_modules
																																				):
																																										pass
																																				else:
																																										_lowercase       :	List[Any]           =   bnb.nn.Linearabit(
																																										    _UpperCAmelCase ,			_UpperCAmelCase ,			module.bias is not None ,			quantization_config.bnb_abit_compute_dtype ,			compress_statistics=quantization_config.bnb_abit_use_double_quant ,			quant_type=quantization_config.bnb_abit_quant_type ,			)
																																										_lowercase       :	Optional[int]           =   True
                    # Store the module class in case we need to transpose the weight later
																														_lowercase       :	str           =   type(_UpperCAmelCase  )
																														# Force requires grad to False to avoid unexpected errors
																														model._modules[name].requires_grad_(_UpperCAmelCase  )
												if len(list(module.children()  )  ) > 0:
																		_lowercase      ,    _lowercase       :	List[str]           =   _replace_with_bnb_linear(
																		    _UpperCAmelCase ,			_UpperCAmelCase ,			_UpperCAmelCase ,			_UpperCAmelCase ,			has_been_replaced=_UpperCAmelCase ,			)
												# Remove the last key for recursion
												current_key_name.pop(-1  )
						return model, has_been_replaced



def 		UpperCamelCase     (       _UpperCAmelCase  :   Optional[int] ,			_UpperCAmelCase  :   Tuple=None ,			_UpperCAmelCase  :   Optional[int]=None ,			_UpperCAmelCase  :   int=None  )   ->			Optional[int]:






						'''simple docstring'''

						_lowercase       :	str           =   ["lm_head"] if modules_to_not_convert is None else modules_to_not_convert
						_lowercase      ,    _lowercase       :	Any           =   _replace_with_bnb_linear(
						    _UpperCAmelCase ,			_UpperCAmelCase ,			_UpperCAmelCase ,			_UpperCAmelCase  )

						if not has_been_replaced:
												logger.warning(
												    "You are loading your model in 8bit or 4bit but no linear modules were found in your model."
												    " Please double check your model architecture, or submit an issue on github if you think this is"
												    " a bug."  )

						return model



def 		UpperCamelCase     (       *_UpperCAmelCase  :   Optional[Any] ,			**_UpperCAmelCase  :   Dict  )   ->			Tuple:






						'''simple docstring'''

						warnings.warn(
						    "`replace_8bit_linear` will be deprecated in a future version, please use `replace_with_bnb_linear` instead" ,			_UpperCAmelCase ,			)
						return replace_with_bnb_linear(*_UpperCAmelCase ,			**_UpperCAmelCase  )



def 		UpperCamelCase     (       *_UpperCAmelCase  :   Dict ,			**_UpperCAmelCase  :   Dict  )   ->			int:






						'''simple docstring'''

						warnings.warn(
						    "`set_module_8bit_tensor_to_device` will be deprecated in a future version, please use `set_module_quantized_tensor_to_device` instead" ,			_UpperCAmelCase ,			)
						return set_module_quantized_tensor_to_device(*_UpperCAmelCase ,			**_UpperCAmelCase  )



def 		UpperCamelCase     (       _UpperCAmelCase  :   List[Any]  )   ->			Tuple:






						'''simple docstring'''

						_lowercase       :	Tuple           =   deepcopy(_UpperCAmelCase  )  # this has 0 cost since it is done inside `init_empty_weights` context manager`
						tied_model.tie_weights()

						_lowercase       :	List[Any]           =   find_tied_parameters(_UpperCAmelCase  )
						# For compatibility with Accelerate < 0.18
						if isinstance(_UpperCAmelCase ,			_UpperCAmelCase  ):
												_lowercase       :	Any           =   sum(list(tied_params.values()  ) ,			[]  ) + list(tied_params.keys()  )
						else:
												_lowercase       :	Tuple           =   sum(_UpperCAmelCase ,			[]  )
						_lowercase       :	List[Any]           =   len(_UpperCAmelCase  ) > 0

						# Check if it is a base model
						_lowercase       :	Any           =   not hasattr(_UpperCAmelCase ,			model.base_model_prefix  )

						# Ignore this for base models (BertModel, GPT2Model, etc.)
						if (not has_tied_params) and is_base_model:
												return []

						# otherwise they have an attached head
						_lowercase       :	int           =   list(model.named_children()  )
						_lowercase       :	str           =   [list_modules[-1][0]]

						# add last module together with tied weights
						_lowercase       :	Optional[int]           =   set(_UpperCAmelCase  ) - set(_UpperCAmelCase  )
						_lowercase       :	Optional[int]           =   list(set(_UpperCAmelCase  )  ) + list(_UpperCAmelCase  )

						# remove ".weight" from the keys
						_lowercase       :	str           =   [".weight", ".bias"]
						_lowercase       :	Optional[Any]           =   []
						for name in list_untouched:
												for name_to_remove in names_to_remove:
																		if name_to_remove in name:
																								_lowercase       :	Optional[Any]           =   name.replace(_UpperCAmelCase ,			""  )
												filtered_module_names.append(_UpperCAmelCase  )

						return filtered_module_names




