"""simple docstring"""



import os
import posixpath
import uuid
from dataclasses import dataclass
from typing import TYPE_CHECKING, Iterable, List, Optional, Tuple, Union

import numpy as np
import pyarrow as pa

import datasets
from datasets.arrow_writer import ArrowWriter, ParquetWriter
from datasets.config import MAX_SHARD_SIZE
from datasets.filesystems import (
    is_remote_filesystem,
    rename,
)
from datasets.iterable_dataset import _BaseExamplesIterable
from datasets.utils.py_utils import convert_file_size_to_int


_UpperCamelCase								=		datasets.utils.logging.get_logger(__name__)

if TYPE_CHECKING:
				import pyspark
@dataclass
class 						__UpperCAmelCase    (datasets.BuilderConfig     ):







			'''simple docstring'''
			_UpperCamelCase  :						Optional[datasets.Features]					     =				None





def  _A(		lowerCAmelCase			,  lowerCAmelCase			,  ):
							import pyspark

							def generate_fn():
														A__       :     Tuple								=			df.select("""*"""			,  pyspark.sql.functions.spark_partition_id().alias("""part_id"""							)							)
														for partition_id in partition_order:
																					A__       :     Any								=			df_with_partition_id.select("""*"""							).where(F'''part_id = {partition_id}'''							).drop("""part_id"""							)
																					A__       :     Optional[Any]								=			partition_df.collect()
																					A__       :     Any								=			0
																					for row in rows:
																												yield F'''{partition_id}_{row_id}''', row.asDict()
																												row_id += 1

							return generate_fn
class 						__UpperCAmelCase    (_BaseExamplesIterable     ):







			'''simple docstring'''





			def __init__(			self   ,      snake_case_   ,      snake_case_=None   ,      ):







										'''simple docstring'''


										A__       :     str								=			df
										A__       :     List[str]								=			partition_order or range(self.df.rdd.getNumPartitions()							)
										A__       :     Union[str, Any]								=			_generate_iterable_examples(self.df   ,      self.partition_order							)





			def __iter__(			self							):







										'''simple docstring'''


										yield from self.generate_examples_fn()





			def 		lowerCamelCase					(			self   ,      snake_case_							):







										'''simple docstring'''


										A__       :     Tuple								=			list(range(self.df.rdd.getNumPartitions()							)							)
										generator.shuffle(snake_case_							)
										return SparkExamplesIterable(self.df   ,      partition_order=snake_case_							)





			def 		lowerCamelCase					(			self   ,      snake_case_   ,      snake_case_							):







										'''simple docstring'''


										A__       :     Optional[Any]								=			self.split_shard_indices_by_worker(snake_case_   ,      snake_case_							)
										return SparkExamplesIterable(self.df   ,      partition_order=snake_case_							)





			@property
			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										return len(self.partition_order							)





class 						__UpperCAmelCase    (datasets.DatasetBuilder     ):







			'''simple docstring'''
			_UpperCamelCase  :						Tuple					     =				SparkConfig





			def __init__(			self   ,      snake_case_   ,      snake_case_ = None   ,      snake_case_ = None   ,      **snake_case_   ,      ):







										'''simple docstring'''


										import pyspark

										A__       :     Optional[Any]								=			pyspark.sql.SparkSession.builder.getOrCreate()
										A__       :     str								=			df
										A__       :     Tuple								=			working_dir

										super().__init__(
										    cache_dir=snake_case_   ,      config_name=str(self.df.semanticHash()							)   ,      **snake_case_   ,      )





			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										def create_cache_and_write_probe(snake_case_							):
																	# makedirs with exist_ok will recursively create the directory. It will not throw an error if directories
																	# already exist.
																	os.makedirs(self._cache_dir   ,      exist_ok=snake_case_							)
																	A__       :     str								=			os.path.join(self._cache_dir   ,      """fs_test""" + uuid.uuida().hex							)
																	# Opening the file in append mode will create a new file unless it already exists, in which case it will not
																	# change the file contents.
																	open(snake_case_   ,      """a"""							)
																	return [probe_file]

										if self._spark.conf.get("""spark.master"""   ,      """"""							).startswith("""local"""							):
																	return

										# If the cluster is multi-node, make sure that the user provided a cache_dir and that it is on an NFS
										# accessible to the driver.
										# TODO: Stream batches to the driver using ArrowCollectSerializer instead of throwing an error.
										if self._cache_dir:
																	A__       :     List[Any]								=			(
																	    self._spark.sparkContext.parallelize(range(1							)   ,      1							).mapPartitions(snake_case_							).collect()
																	)
																	if os.path.isfile(probe[0]							):
																								return

										raise ValueError(
										    """When using Dataset.from_spark on a multi-node cluster, the driver and all workers should be able to access cache_dir"""							)





			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										return datasets.DatasetInfo(features=self.config.features							)





			def 		lowerCamelCase					(			self   ,      snake_case_							):







										'''simple docstring'''


										return [datasets.SplitGenerator(name=datasets.Split.TRAIN							)]





			def 		lowerCamelCase					(			self   ,      snake_case_							):







										'''simple docstring'''


										import pyspark

										def get_arrow_batch_size(snake_case_							):
																	for batch in it:
																								yield pa.RecordBatch.from_pydict({"""batch_bytes""": [batch.nbytes]}							)

										A__       :     Any								=			self.df.count()
										A__       :     int								=			df_num_rows if df_num_rows <= 100 else 100
										# Approximate the size of each row (in Arrow format) by averaging over a max-100-row sample.
										A__       :     List[Any]								=			(
										    self.df.limit(snake_case_							)
										    .repartition(1							)
										    .mapInArrow(snake_case_   ,      """batch_bytes: long"""							)
										    .agg(pyspark.sql.functions.sum("""batch_bytes"""							).alias("""sample_bytes"""							)							)
										    .collect()[0]
										    .sample_bytes
										    / sample_num_rows
										)
										A__       :     Dict								=			approx_bytes_per_row * df_num_rows
										if approx_total_size > max_shard_size:
																	# Make sure there is at least one row per partition.
																	A__       :     List[str]								=			min(snake_case_   ,      int(approx_total_size / max_shard_size							)							)
																	A__       :     Tuple								=			self.df.repartition(snake_case_							)





			def 		lowerCamelCase					(			self   ,      snake_case_   ,      snake_case_   ,      snake_case_   ,      ):







										'''simple docstring'''


										import pyspark

										A__       :     List[Any]								=			ParquetWriter if file_format == """parquet""" else ArrowWriter
										A__       :     Dict								=			os.path.join(self._working_dir   ,      os.path.basename(snake_case_							)							) if self._working_dir else fpath
										A__       :     Dict								=			file_format == """parquet"""

										# Define these so that we don't reference self in write_arrow, which will result in a pickling error due to
										# pickling the SparkContext.
										A__       :     Optional[int]								=			self.config.features
										A__       :     str								=			self._writer_batch_size
										A__       :     Optional[Any]								=			self._fs.storage_options

										def write_arrow(snake_case_							):
																	# Within the same SparkContext, no two task attempts will share the same attempt ID.
																	A__       :     Union[str, Any]								=			pyspark.TaskContext().taskAttemptId()
																	A__       :     Dict								=			next(snake_case_   ,      snake_case_							)
																	if first_batch is None:
																								# Some partitions might not receive any data.
																								return pa.RecordBatch.from_arrays(
																								    [[task_id], [0], [0]]   ,      names=["""task_id""", """num_examples""", """num_bytes"""]   ,      )
																	A__       :     Dict								=			0
																	A__       :     Tuple								=			writer_class(
																	    features=snake_case_   ,      path=working_fpath.replace("""SSSSS"""   ,      F'''{shard_id:05d}'''							).replace("""TTTTT"""   ,      F'''{task_id:05d}'''							)   ,      writer_batch_size=snake_case_   ,      storage_options=snake_case_   ,      embed_local_files=snake_case_   ,      )
																	A__       :     Union[str, Any]								=			pa.Table.from_batches([first_batch]							)
																	writer.write_table(snake_case_							)
																	for batch in it:
																								if max_shard_size is not None and writer._num_bytes >= max_shard_size:
																															A__				,							A__       :     str								=			writer.finalize()
																															writer.close()
																															yield pa.RecordBatch.from_arrays(
																															    [[task_id], [num_examples], [num_bytes]]   ,      names=["""task_id""", """num_examples""", """num_bytes"""]   ,      )
																															shard_id += 1
																															A__       :     Tuple								=			writer_class(
																															    features=writer._features   ,      path=working_fpath.replace("""SSSSS"""   ,      F'''{shard_id:05d}'''							).replace("""TTTTT"""   ,      F'''{task_id:05d}'''							)   ,      writer_batch_size=snake_case_   ,      storage_options=snake_case_   ,      embed_local_files=snake_case_   ,      )
																								A__       :     List[str]								=			pa.Table.from_batches([batch]							)
																								writer.write_table(snake_case_							)

																	if writer._num_bytes > 0:
																								A__				,							A__       :     Union[str, Any]								=			writer.finalize()
																								writer.close()
																								yield pa.RecordBatch.from_arrays(
																								    [[task_id], [num_examples], [num_bytes]]   ,      names=["""task_id""", """num_examples""", """num_bytes"""]   ,      )

																	if working_fpath != fpath:
																								for file in os.listdir(os.path.dirname(snake_case_							)							):
																															A__       :     List[Any]								=			os.path.join(os.path.dirname(snake_case_							)   ,      os.path.basename(snake_case_							)							)
																															shutil.move(snake_case_   ,      snake_case_							)

										A__       :     str								=			(
										    self.df.mapInArrow(snake_case_   ,      """task_id: long, num_examples: long, num_bytes: long"""							)
										    .groupBy("""task_id"""							)
										    .agg(
										        pyspark.sql.functions.sum("""num_examples"""							).alias("""total_num_examples"""							)   ,      pyspark.sql.functions.sum("""num_bytes"""							).alias("""total_num_bytes"""							)   ,      pyspark.sql.functions.count("""num_bytes"""							).alias("""num_shards"""							)   ,      pyspark.sql.functions.collect_list("""num_examples"""							).alias("""shard_lengths"""							)   ,      )
										    .collect()
										)
										for row in stats:
																	yield row.task_id, (row.total_num_examples, row.total_num_bytes, row.num_shards, row.shard_lengths)





			def 		lowerCamelCase					(			self   ,      snake_case_   ,      snake_case_ = "arrow"   ,      snake_case_ = None   ,      snake_case_ = None   ,      **snake_case_   ,      ):







										'''simple docstring'''


										self._validate_cache_dir()

										A__       :     List[Any]								=			convert_file_size_to_int(max_shard_size or MAX_SHARD_SIZE							)
										self._repartition_df_if_needed(snake_case_							)
										A__       :     Any								=			not is_remote_filesystem(self._fs							)
										A__       :     Optional[int]								=			os.path.join if is_local else posixpath.join

										A__       :     Tuple								=			"""-TTTTT-SSSSS-of-NNNNN"""
										A__       :     Optional[Any]								=			F'''{self.name}-{split_generator.name}{SUFFIX}.{file_format}'''
										A__       :     List[Any]								=			path_join(self._output_dir   ,      snake_case_							)

										A__       :     Optional[Any]								=			0
										A__       :     Dict								=			0
										A__       :     str								=			0
										A__       :     Optional[Any]								=			[]
										A__       :     Any								=			[]

										for task_id, content in self._prepare_split_single(snake_case_   ,      snake_case_   ,      snake_case_							):
																	(
																	    (
																	    A__
																	)				,							(
																	    A__
																	)				,							(
																	    A__
																	)				,							(
																	    A__
																	)				,							
																	)       :     Optional[Any]								=			content
																	if num_bytes > 0:
																								total_num_examples += num_examples
																								total_num_bytes += num_bytes
																								total_shards += num_shards
																								task_id_and_num_shards.append((task_id, num_shards)							)
																								all_shard_lengths.extend(snake_case_							)

										A__       :     Optional[Any]								=			total_num_examples
										A__       :     List[str]								=			total_num_bytes

										# should rename everything at the end
										logger.debug(F'''Renaming {total_shards} shards.'''							)
										if total_shards > 1:
																	A__       :     List[Any]								=			all_shard_lengths

																	# Define fs outside of _rename_shard so that we don't reference self in the function, which will result in a
																	# pickling error due to pickling the SparkContext.
																	A__       :     Optional[Any]								=			self._fs

																	# use the -SSSSS-of-NNNNN pattern
																	def _rename_shard(
																	    snake_case_   ,      snake_case_   ,      snake_case_   ,      ):
																								rename(
																								    snake_case_   ,      fpath.replace("""SSSSS"""   ,      F'''{shard_id:05d}'''							).replace("""TTTTT"""   ,      F'''{task_id:05d}'''							)   ,      fpath.replace("""TTTTT-SSSSS"""   ,      F'''{global_shard_id:05d}'''							).replace("""NNNNN"""   ,      F'''{total_shards:05d}'''							)   ,      )

																	A__       :     List[Any]								=			[]
																	A__       :     Optional[int]								=			0
																	for i in range(len(snake_case_							)							):
																								A__				,							A__       :     Optional[int]								=			task_id_and_num_shards[i]
																								for shard_id in range(snake_case_							):
																															args.append([task_id, shard_id, global_shard_id]							)
																															global_shard_id += 1
																	self._spark.sparkContext.parallelize(snake_case_   ,      len(snake_case_							)							).map(lambda snake_case_							: _rename_shard(*snake_case_							)							).collect()
										else:
																	# don't use any pattern
																	A__       :     str								=			0
																	A__       :     Tuple								=			task_id_and_num_shards[0][0]
																	self._rename(
																	    fpath.replace("""SSSSS"""   ,      F'''{shard_id:05d}'''							).replace("""TTTTT"""   ,      F'''{task_id:05d}'''							)   ,      fpath.replace(snake_case_   ,      """"""							)   ,      )





			def 		lowerCamelCase					(			self   ,      snake_case_   ,      ):







										'''simple docstring'''


										return SparkExamplesIterable(self.df							)






"""simple docstring"""



from collections import OrderedDict
from typing import Any, Mapping, Optional

from ... import PreTrainedTokenizer, TensorType, is_torch_available
from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfigWithPast
from ...utils import logging


_UpperCamelCase								=		logging.get_logger(__name__)

_UpperCamelCase								=		{
    "EleutherAI/gpt-neo-1.3B": "https://huggingface.co/EleutherAI/gpt-neo-1.3B/resolve/main/config.json",
    # See all GPTNeo models at https://huggingface.co/models?filter=gpt_neo
}
class 						__UpperCAmelCase    (__A     ):







			'''simple docstring'''
			_UpperCamelCase  :						Tuple					     =				'gpt_neo'
			_UpperCamelCase  :						Optional[Any]					     =				['past_key_values']
			_UpperCamelCase  :						int					     =				{'num_attention_heads': 'num_heads', 'num_hidden_layers': 'num_layers'}





			def __init__(			self   ,      snake_case_=50_257   ,      snake_case_=2_048   ,      snake_case_=2_048   ,      snake_case_=24   ,      snake_case_=[[["global", "local"], 12]]   ,      snake_case_=16   ,      snake_case_=None   ,      snake_case_=256   ,      snake_case_="gelu_new"   ,      snake_case_=0.0   ,      snake_case_=0.0   ,      snake_case_=0.0   ,      snake_case_=0.1   ,      snake_case_=1E-5   ,      snake_case_=0.02   ,      snake_case_=True   ,      snake_case_=50_256   ,      snake_case_=50_256   ,      **snake_case_   ,      ):







										'''simple docstring'''


										A__       :     int								=			vocab_size
										A__       :     Optional[Any]								=			max_position_embeddings
										A__       :     int								=			hidden_size
										A__       :     List[Any]								=			num_layers
										A__       :     Any								=			num_heads
										A__       :     List[str]								=			intermediate_size
										A__       :     Dict								=			window_size
										A__       :     Optional[int]								=			activation_function
										A__       :     Optional[int]								=			resid_dropout
										A__       :     List[str]								=			embed_dropout
										A__       :     str								=			attention_dropout
										A__       :     List[str]								=			classifier_dropout
										A__       :     str								=			layer_norm_epsilon
										A__       :     str								=			initializer_range
										A__       :     Any								=			use_cache

										A__       :     List[str]								=			bos_token_id
										A__       :     Any								=			eos_token_id

										A__       :     Optional[Any]								=			attention_types
										A__       :     List[Any]								=			self.expand_attention_types_params(snake_case_							)

										if len(self.attention_layers							) != self.num_layers:
																	raise ValueError(
																	    """Configuration for convolutional module is incorrect. """
																	    """It is required that `len(config.attention_layers)` == `config.num_layers` """
																	    F'''but is `len(config.attention_layers) = {len(self.attention_layers							)}`, '''
																	    F'''`config.num_layers = {self.num_layers}`. '''
																	    """`config.attention_layers` is prepared using `config.attention_types`. """
																	    """Please verify the value of `config.attention_types` argument."""							)

										super().__init__(bos_token_id=snake_case_   ,      eos_token_id=snake_case_   ,      **snake_case_							)





			@staticmethod
			def 		lowerCamelCase					(			snake_case_							):







										'''simple docstring'''


										A__       :     Optional[Any]								=			[]
										for item in attention_types:
																	for _ in range(item[1]							):
																								attentions.extend(item[0]							)
										return attentions





def  _A(		lowerCAmelCase			,  lowerCAmelCase			,  lowerCAmelCase			,  lowerCAmelCase							):
							import torch

							A__       :     Union[str, Any]								=			input.size()
							A__       :     str								=			len(lowerCAmelCase							)
							A__       :     Union[str, Any]								=			shape[dimension]

							A__       :     Dict								=			torch.arange(0			,  lowerCAmelCase			,  lowerCAmelCase							)
							A__       :     Dict								=			torch.div(sizedim - size			,  lowerCAmelCase			,  rounding_mode="""floor"""							) + 1
							A__       :     str								=			torch.arange(lowerCAmelCase							) + low_indices[:min_length][:, None]

							A__       :     str								=			[slice(lowerCAmelCase							)] * rank
							A__       :     Optional[Any]								=			indices
							A__       :     Tuple								=			input[s]

							A__       :     List[str]								=			list(range(0			,  rank + 1							)							)
							perm.append(perm.pop(dimension + 1							)							)

							return sliced.permute(lowerCAmelCase							)

def  _A(		lowerCAmelCase			,  lowerCAmelCase							):
							import torch

							A__       :     Optional[Any]								=			torch.arange(1			,  lowerCAmelCase							)
							A__       :     int								=			torch.remainder(lowerCAmelCase			,  lowerCAmelCase							)
							A__       :     str								=			remainders == 0
							A__       :     Tuple								=			candidates[divisor_indices]
							A__       :     Any								=			torch.max(lowerCAmelCase							)
							return largest_divisor, torch.div(lowerCAmelCase			,  lowerCAmelCase			,  rounding_mode="""floor"""							)



class 						__UpperCAmelCase    (__A     ):







			'''simple docstring'''





			@property
			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										A__       :     Any								=			OrderedDict({"""input_ids""": {0: """batch""", 1: """sequence"""}}							)
										if self.use_past:
																	self.fill_with_past_key_values_(snake_case_   ,      direction="""inputs"""							)
																	A__       :     str								=			{0: """batch""", 1: """past_sequence + sequence"""}
										else:
																	A__       :     Optional[Any]								=			{0: """batch""", 1: """sequence"""}

										return common_inputs





			@property
			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										return self._config.num_heads





			def 		lowerCamelCase					(			self   ,      snake_case_   ,      snake_case_ = -1   ,      snake_case_ = -1   ,      snake_case_ = False   ,      snake_case_ = None   ,      ):







										'''simple docstring'''


										A__       :     Optional[int]								=			super(snake_case_   ,      self							).generate_dummy_inputs(
										    snake_case_   ,      batch_size=snake_case_   ,      seq_length=snake_case_   ,      is_pair=snake_case_   ,      framework=snake_case_							)

										# We need to order the input in the way they appears in the forward()
										A__       :     List[Any]								=			OrderedDict({"""input_ids""": common_inputs["""input_ids"""]}							)

										# Need to add the past_keys
										if self.use_past:
																	if not is_torch_available():
																								raise ValueError("""Cannot generate dummy past_keys inputs without PyTorch installed."""							)
																	else:
																								import torch

																								A__				,							A__       :     Dict								=			common_inputs["""input_ids"""].shape
																								# Not using the same length for past_key_values
																								A__       :     Any								=			seqlen + 2
																								A__       :     Any								=			(
																								    batch,
																								    self.num_attention_heads,
																								    past_key_values_length,
																								    self._config.hidden_size // self.num_attention_heads,
																								)
																								A__       :     List[Any]								=			[
																								    (torch.zeros(snake_case_							), torch.zeros(snake_case_							)) for _ in range(self.num_layers							)
																								]

										A__       :     Tuple								=			common_inputs["""attention_mask"""]
										if self.use_past:
																	A__       :     List[Any]								=			ordered_inputs["""attention_mask"""].dtype
																	A__       :     Optional[Any]								=			torch.cat(
																	    [ordered_inputs["""attention_mask"""], torch.ones(snake_case_   ,      snake_case_   ,      dtype=snake_case_							)]   ,      dim=1							)

										return ordered_inputs





			@property
			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										return 13





