






"""simple docstring"""







import json
import os
import tempfile

import datasets
from datasets.arrow_writer import ArrowWriter
from datasets.features import ArrayaD
from utils import generate_examples, get_duration


lowerCAmelCase_		:			Optional[int]				   =					(3_0, 4_8_7)
lowerCAmelCase_		:			Tuple				   =					(3_6, 1_0_2_4)
lowerCAmelCase_		:			str				   =					(1_0_0, 1_0_0)
lowerCAmelCase_		:			Union[str, Any]				   =					1_0_0

lowerCAmelCase_		:			Any				   =					datasets.Features(
    {'''text''': ArrayaD(SHAPE_TEST_1, dtype='''float32'''), '''image''': ArrayaD(SHAPE_TEST_2, dtype='''float32''')}
)

lowerCAmelCase_   ,    lowerCAmelCase_		:			str				   =					os.path.split(__file__)
lowerCAmelCase_		:			Tuple				   =					os.path.join(RESULTS_BASEPATH, '''results''', RESULTS_FILENAME.replace('''.py''', '''.json'''))






@get_duration
def 					_lowerCAmelCase				(					lowerCAmelCase		,					lowerCAmelCase		,					lowerCAmelCase						):







       '''simple docstring'''
       with ArrowWriter(features=lowerCAmelCase		,					path=os.path.join(lowerCAmelCase		,					"""beta.arrow"""						)						) as writer:
              for key, record in dummy_data:
                     UpperCAmelCase					=	my_features.encode_example(lowerCAmelCase						)
                     writer.write(lowerCAmelCase						)
              UpperCAmelCase			,				UpperCAmelCase					=	writer.finalize()






@get_duration
def 					_lowerCAmelCase				(					lowerCAmelCase		,					lowerCAmelCase						):







       '''simple docstring'''
       UpperCAmelCase					=	datasets.Dataset.from_file(
           filename=os.path.join(lowerCAmelCase		,					"""beta.arrow"""						)		,					info=datasets.DatasetInfo(features=lowerCAmelCase						)						)
       for _ in dataset:
              pass






@get_duration
def 					_lowerCAmelCase				(					lowerCAmelCase		,					lowerCAmelCase						):







       '''simple docstring'''
       UpperCAmelCase					=	datasets.Dataset.from_file(
           filename=os.path.join(lowerCAmelCase		,					"""beta.arrow"""						)		,					info=datasets.DatasetInfo(features=lowerCAmelCase						)						)
       dataset.set_format("""numpy"""						)
       for _ in dataset:
              pass






@get_duration
def 					_lowerCAmelCase				(					lowerCAmelCase		,					lowerCAmelCase						):







       '''simple docstring'''
       UpperCAmelCase					=	10
       UpperCAmelCase					=	datasets.Dataset.from_file(
           filename=os.path.join(lowerCAmelCase		,					"""beta.arrow"""						)		,					info=datasets.DatasetInfo(features=lowerCAmelCase						)						)
       for i in range(0		,					len(lowerCAmelCase						)		,					lowerCAmelCase						):
              UpperCAmelCase					=	dataset[i : i + batch_size]






@get_duration
def 					_lowerCAmelCase				(					lowerCAmelCase		,					lowerCAmelCase						):







       '''simple docstring'''
       UpperCAmelCase					=	10
       UpperCAmelCase					=	datasets.Dataset.from_file(
           filename=os.path.join(lowerCAmelCase		,					"""beta.arrow"""						)		,					info=datasets.DatasetInfo(features=lowerCAmelCase						)						)
       dataset.set_format("""numpy"""						)
       for i in range(0		,					len(lowerCAmelCase						)		,					lowerCAmelCase						):
              UpperCAmelCase					=	dataset[i : i + batch_size]






@get_duration
def 					_lowerCAmelCase				(					lowerCAmelCase		,					lowerCAmelCase						):







       '''simple docstring'''
       UpperCAmelCase					=	datasets.Dataset.from_file(
           filename=os.path.join(lowerCAmelCase		,					"""beta.arrow"""						)		,					info=datasets.DatasetInfo(features=lowerCAmelCase						)						)
       for col in feats:
              UpperCAmelCase					=	dataset[col]






@get_duration
def 					_lowerCAmelCase				(					lowerCAmelCase		,					lowerCAmelCase						):







       '''simple docstring'''
       UpperCAmelCase					=	datasets.Dataset.from_file(
           filename=os.path.join(lowerCAmelCase		,					"""beta.arrow"""						)		,					info=datasets.DatasetInfo(features=lowerCAmelCase						)						)
       dataset.set_format("""numpy"""						)
       for col in feats:
              UpperCAmelCase					=	dataset[col]






def 					_lowerCAmelCase				(					):







       '''simple docstring'''
       UpperCAmelCase					=	{}
       UpperCAmelCase					=	(
           read_unformated,
           read_formatted_as_numpy,
           read_batch_unformated,
           read_batch_formatted_as_numpy,
           read_col_unformated,
           read_col_formatted_as_numpy,
       )
       with tempfile.TemporaryDirectory() as tmp_dir:
              UpperCAmelCase					=	datasets.Features({"""image""": ArrayaD(lowerCAmelCase		,					dtype="""float32"""						)}						)
              UpperCAmelCase					=	generate_examples(features=lowerCAmelCase		,					num_examples=lowerCAmelCase						)
              UpperCAmelCase					=	write(lowerCAmelCase		,					lowerCAmelCase		,					lowerCAmelCase						)
              for read_func in read_functions:
                     UpperCAmelCase					=	read_func(lowerCAmelCase		,					lowerCAmelCase						)

       with tempfile.TemporaryDirectory() as tmp_dir:
              # don't use fixed length for fair comparison
              # feats = datasets.Features(
              #     {"image": datasets.Sequence(datasets.Sequence(datasets.Value("float32"), SPEED_TEST_SHAPE[1]), SPEED_TEST_SHAPE[0])}
              # )
              UpperCAmelCase					=	datasets.Features({"""image""": datasets.Sequence(datasets.Sequence(datasets.Value("""float32"""						)						)						)}						)
              UpperCAmelCase					=	generate_examples(
                  features=lowerCAmelCase		,					num_examples=lowerCAmelCase		,					seq_shapes={"""image""": SPEED_TEST_SHAPE}						)
              UpperCAmelCase					=	write(lowerCAmelCase		,					lowerCAmelCase		,					lowerCAmelCase						)
              for read_func in read_functions:
                     UpperCAmelCase					=	read_func(lowerCAmelCase		,					lowerCAmelCase						)

       with tempfile.TemporaryDirectory() as tmp_dir:
              # don't use fixed length for fair comparison
              # feats = datasets.Features(
              #     {"image": datasets.Sequence(datasets.Value("float32"), SPEED_TEST_SHAPE[0] * SPEED_TEST_SHAPE[1])}
              # )
              UpperCAmelCase					=	datasets.Features({"""image""": datasets.Sequence(datasets.Value("""float32"""						)						)}						)
              UpperCAmelCase					=	generate_examples(
                  features=lowerCAmelCase		,					num_examples=lowerCAmelCase		,					seq_shapes={"""image""": [SPEED_TEST_SHAPE[0] * SPEED_TEST_SHAPE[1]]}		,					)
              UpperCAmelCase					=	write(lowerCAmelCase		,					lowerCAmelCase		,					lowerCAmelCase						)
              for read_func in read_functions:
                     UpperCAmelCase					=	read_func(lowerCAmelCase		,					lowerCAmelCase						)

       with open(lowerCAmelCase		,					"""wb"""						) as f:
              f.write(json.dumps(lowerCAmelCase						).encode("""utf-8"""						)						)


if __name__ == "__main__":  # useful to run the profiler
 benchmark_array_xd()






import unittest

import numpy as np

from transformers import ElectraConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask


if is_flax_available():
							from transformers.models.electra.modeling_flax_electra import (
							    FlaxElectraForCausalLM,
							    FlaxElectraForMaskedLM,
							    FlaxElectraForMultipleChoice,
							    FlaxElectraForPreTraining,
							    FlaxElectraForQuestionAnswering,
							    FlaxElectraForSequenceClassification,
							    FlaxElectraForTokenClassification,
							    FlaxElectraModel,
							)


class 				__SCREAMING_SNAKE_CASE							(		unittest.TestCase  ):

					def __init__(				self ,       __lowercase ,       __lowercase=13 ,       __lowercase=7 ,       __lowercase=True ,       __lowercase=True ,       __lowercase=True ,       __lowercase=True ,       __lowercase=99 ,       __lowercase=24 ,       __lowercase=32 ,       __lowercase=5 ,       __lowercase=4 ,       __lowercase=37 ,       __lowercase="gelu" ,       __lowercase=0.1 ,       __lowercase=0.1 ,       __lowercase=512 ,       __lowercase=16 ,       __lowercase=2 ,       __lowercase=0.02 ,       __lowercase=4 ,       ):







						'''simple docstring'''




						lowerCamelCase_							:		str							   =							parent
						lowerCamelCase_							:		Any							   =							batch_size
						lowerCamelCase_							:		Union[str, Any]							   =							seq_length
						lowerCamelCase_							:		Dict							   =							is_training
						lowerCamelCase_							:		Any							   =							use_attention_mask
						lowerCamelCase_							:		Any							   =							use_token_type_ids
						lowerCamelCase_							:		Any							   =							use_labels
						lowerCamelCase_							:		Any							   =							vocab_size
						lowerCamelCase_							:		Any							   =							hidden_size
						lowerCamelCase_							:		Optional[Any]							   =							embedding_size
						lowerCamelCase_							:		Dict							   =							num_hidden_layers
						lowerCamelCase_							:		Optional[Any]							   =							num_attention_heads
						lowerCamelCase_							:		Optional[Any]							   =							intermediate_size
						lowerCamelCase_							:		List[str]							   =							hidden_act
						lowerCamelCase_							:		Optional[Any]							   =							hidden_dropout_prob
						lowerCamelCase_							:		int							   =							attention_probs_dropout_prob
						lowerCamelCase_							:		Optional[int]							   =							max_position_embeddings
						lowerCamelCase_							:		List[str]							   =							type_vocab_size
						lowerCamelCase_							:		int							   =							type_sequence_label_size
						lowerCamelCase_							:		Tuple							   =							initializer_range
						lowerCamelCase_							:		int							   =							num_choices

					def 	lowercase_							(				self					):







						'''simple docstring'''




						lowerCamelCase_							:		int							   =							ids_tensor([self.batch_size, self.seq_length] ,       self.vocab_size					)

						lowerCamelCase_							:		List[Any]							   =							None
						if self.use_attention_mask:
							lowerCamelCase_							:		Tuple							   =							random_attention_mask([self.batch_size, self.seq_length]					)

						lowerCamelCase_							:		int							   =							None
						if self.use_token_type_ids:
							lowerCamelCase_							:		List[Any]							   =							ids_tensor([self.batch_size, self.seq_length] ,       self.type_vocab_size					)

						lowerCamelCase_							:		List[str]							   =							ElectraConfig(
						    vocab_size=self.vocab_size ,       hidden_size=self.hidden_size ,       embedding_size=self.embedding_size ,       num_hidden_layers=self.num_hidden_layers ,       num_attention_heads=self.num_attention_heads ,       intermediate_size=self.intermediate_size ,       hidden_act=self.hidden_act ,       hidden_dropout_prob=self.hidden_dropout_prob ,       attention_probs_dropout_prob=self.attention_probs_dropout_prob ,       max_position_embeddings=self.max_position_embeddings ,       type_vocab_size=self.type_vocab_size ,       initializer_range=self.initializer_range ,       )

						return config, input_ids, token_type_ids, attention_mask





					def 	lowercase_							(				self					):







						'''simple docstring'''




						lowerCamelCase_							:		Tuple							   =							self.prepare_config_and_inputs()
						lowerCamelCase_   ,			lowerCamelCase_   ,			lowerCamelCase_   ,			lowerCamelCase_							:		Tuple							   =							config_and_inputs
						lowerCamelCase_							:		Union[str, Any]							   =							{'input_ids': input_ids, 'token_type_ids': token_type_ids, 'attention_mask': attention_mask}
						return config, inputs_dict


@require_flax
class 				__SCREAMING_SNAKE_CASE							(		lowerCamelCase   ,     unittest.TestCase  ):
					_lowerCamelCase		  = True

					_lowerCamelCase		  = (
					    (
					        FlaxElectraModel,
					        FlaxElectraForCausalLM,
					        FlaxElectraForMaskedLM,
					        FlaxElectraForPreTraining,
					        FlaxElectraForTokenClassification,
					        FlaxElectraForQuestionAnswering,
					        FlaxElectraForMultipleChoice,
					        FlaxElectraForSequenceClassification,
					    )
					    if is_flax_available()
					    else ()
					)

					def 	lowercase_							(				self					):







						'''simple docstring'''




						lowerCamelCase_							:		Union[str, Any]							   =							FlaxElectraModelTester(self					)





					@slow
					def 	lowercase_							(				self					):







						'''simple docstring'''




						for model_class_name in self.all_model_classes:
							if model_class_name == FlaxElectraForMaskedLM:
								lowerCamelCase_							:		Union[str, Any]							   =							model_class_name.from_pretrained('google/electra-small-generator'					)
							else:
								lowerCamelCase_							:		List[Any]							   =							model_class_name.from_pretrained('google/electra-small-discriminator'					)
							lowerCamelCase_							:		Dict							   =							model(np.ones((1, 1)					)					)
							self.assertIsNotNone(__lowercase					)
