






"""simple docstring"""







import json
import os
import tempfile

import datasets
from datasets.arrow_writer import ArrowWriter
from datasets.features import ArrayaD
from utils import generate_examples, get_duration


lowerCAmelCase_		:			Optional[int]				   =					(3_0, 4_8_7)
lowerCAmelCase_		:			Tuple				   =					(3_6, 1_0_2_4)
lowerCAmelCase_		:			str				   =					(1_0_0, 1_0_0)
lowerCAmelCase_		:			Union[str, Any]				   =					1_0_0

lowerCAmelCase_		:			Any				   =					datasets.Features(
    {'''text''': ArrayaD(SHAPE_TEST_1, dtype='''float32'''), '''image''': ArrayaD(SHAPE_TEST_2, dtype='''float32''')}
)

lowerCAmelCase_   ,    lowerCAmelCase_		:			str				   =					os.path.split(__file__)
lowerCAmelCase_		:			Tuple				   =					os.path.join(RESULTS_BASEPATH, '''results''', RESULTS_FILENAME.replace('''.py''', '''.json'''))






@get_duration
def 					_lowerCAmelCase				(					lowerCAmelCase		,					lowerCAmelCase		,					lowerCAmelCase						):







       '''simple docstring'''
       with ArrowWriter(features=lowerCAmelCase		,					path=os.path.join(lowerCAmelCase		,					"""beta.arrow"""						)						) as writer:
              for key, record in dummy_data:
                     UpperCAmelCase					=	my_features.encode_example(lowerCAmelCase						)
                     writer.write(lowerCAmelCase						)
              UpperCAmelCase			,				UpperCAmelCase					=	writer.finalize()






@get_duration
def 					_lowerCAmelCase				(					lowerCAmelCase		,					lowerCAmelCase						):







       '''simple docstring'''
       UpperCAmelCase					=	datasets.Dataset.from_file(
           filename=os.path.join(lowerCAmelCase		,					"""beta.arrow"""						)		,					info=datasets.DatasetInfo(features=lowerCAmelCase						)						)
       for _ in dataset:
              pass






@get_duration
def 					_lowerCAmelCase				(					lowerCAmelCase		,					lowerCAmelCase						):







       '''simple docstring'''
       UpperCAmelCase					=	datasets.Dataset.from_file(
           filename=os.path.join(lowerCAmelCase		,					"""beta.arrow"""						)		,					info=datasets.DatasetInfo(features=lowerCAmelCase						)						)
       dataset.set_format("""numpy"""						)
       for _ in dataset:
              pass






@get_duration
def 					_lowerCAmelCase				(					lowerCAmelCase		,					lowerCAmelCase						):







       '''simple docstring'''
       UpperCAmelCase					=	10
       UpperCAmelCase					=	datasets.Dataset.from_file(
           filename=os.path.join(lowerCAmelCase		,					"""beta.arrow"""						)		,					info=datasets.DatasetInfo(features=lowerCAmelCase						)						)
       for i in range(0		,					len(lowerCAmelCase						)		,					lowerCAmelCase						):
              UpperCAmelCase					=	dataset[i : i + batch_size]






@get_duration
def 					_lowerCAmelCase				(					lowerCAmelCase		,					lowerCAmelCase						):







       '''simple docstring'''
       UpperCAmelCase					=	10
       UpperCAmelCase					=	datasets.Dataset.from_file(
           filename=os.path.join(lowerCAmelCase		,					"""beta.arrow"""						)		,					info=datasets.DatasetInfo(features=lowerCAmelCase						)						)
       dataset.set_format("""numpy"""						)
       for i in range(0		,					len(lowerCAmelCase						)		,					lowerCAmelCase						):
              UpperCAmelCase					=	dataset[i : i + batch_size]






@get_duration
def 					_lowerCAmelCase				(					lowerCAmelCase		,					lowerCAmelCase						):







       '''simple docstring'''
       UpperCAmelCase					=	datasets.Dataset.from_file(
           filename=os.path.join(lowerCAmelCase		,					"""beta.arrow"""						)		,					info=datasets.DatasetInfo(features=lowerCAmelCase						)						)
       for col in feats:
              UpperCAmelCase					=	dataset[col]






@get_duration
def 					_lowerCAmelCase				(					lowerCAmelCase		,					lowerCAmelCase						):







       '''simple docstring'''
       UpperCAmelCase					=	datasets.Dataset.from_file(
           filename=os.path.join(lowerCAmelCase		,					"""beta.arrow"""						)		,					info=datasets.DatasetInfo(features=lowerCAmelCase						)						)
       dataset.set_format("""numpy"""						)
       for col in feats:
              UpperCAmelCase					=	dataset[col]






def 					_lowerCAmelCase				(					):







       '''simple docstring'''
       UpperCAmelCase					=	{}
       UpperCAmelCase					=	(
           read_unformated,
           read_formatted_as_numpy,
           read_batch_unformated,
           read_batch_formatted_as_numpy,
           read_col_unformated,
           read_col_formatted_as_numpy,
       )
       with tempfile.TemporaryDirectory() as tmp_dir:
              UpperCAmelCase					=	datasets.Features({"""image""": ArrayaD(lowerCAmelCase		,					dtype="""float32"""						)}						)
              UpperCAmelCase					=	generate_examples(features=lowerCAmelCase		,					num_examples=lowerCAmelCase						)
              UpperCAmelCase					=	write(lowerCAmelCase		,					lowerCAmelCase		,					lowerCAmelCase						)
              for read_func in read_functions:
                     UpperCAmelCase					=	read_func(lowerCAmelCase		,					lowerCAmelCase						)

       with tempfile.TemporaryDirectory() as tmp_dir:
              # don't use fixed length for fair comparison
              # feats = datasets.Features(
              #     {"image": datasets.Sequence(datasets.Sequence(datasets.Value("float32"), SPEED_TEST_SHAPE[1]), SPEED_TEST_SHAPE[0])}
              # )
              UpperCAmelCase					=	datasets.Features({"""image""": datasets.Sequence(datasets.Sequence(datasets.Value("""float32"""						)						)						)}						)
              UpperCAmelCase					=	generate_examples(
                  features=lowerCAmelCase		,					num_examples=lowerCAmelCase		,					seq_shapes={"""image""": SPEED_TEST_SHAPE}						)
              UpperCAmelCase					=	write(lowerCAmelCase		,					lowerCAmelCase		,					lowerCAmelCase						)
              for read_func in read_functions:
                     UpperCAmelCase					=	read_func(lowerCAmelCase		,					lowerCAmelCase						)

       with tempfile.TemporaryDirectory() as tmp_dir:
              # don't use fixed length for fair comparison
              # feats = datasets.Features(
              #     {"image": datasets.Sequence(datasets.Value("float32"), SPEED_TEST_SHAPE[0] * SPEED_TEST_SHAPE[1])}
              # )
              UpperCAmelCase					=	datasets.Features({"""image""": datasets.Sequence(datasets.Value("""float32"""						)						)}						)
              UpperCAmelCase					=	generate_examples(
                  features=lowerCAmelCase		,					num_examples=lowerCAmelCase		,					seq_shapes={"""image""": [SPEED_TEST_SHAPE[0] * SPEED_TEST_SHAPE[1]]}		,					)
              UpperCAmelCase					=	write(lowerCAmelCase		,					lowerCAmelCase		,					lowerCAmelCase						)
              for read_func in read_functions:
                     UpperCAmelCase					=	read_func(lowerCAmelCase		,					lowerCAmelCase						)

       with open(lowerCAmelCase		,					"""wb"""						) as f:
              f.write(json.dumps(lowerCAmelCase						).encode("""utf-8"""						)						)


if __name__ == "__main__":  # useful to run the profiler
 benchmark_array_xd()








"""simple docstring"""







import unittest

import numpy as np

from transformers import ElectraConfig, is_flax_available
from transformers.testing_utils import require_flax, slow

from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask


if is_flax_available():
 from transformers.models.electra.modeling_flax_electra import (
     FlaxElectraForCausalLM,
     FlaxElectraForMaskedLM,
     FlaxElectraForMultipleChoice,
     FlaxElectraForPreTraining,
     FlaxElectraForQuestionAnswering,
     FlaxElectraForSequenceClassification,
     FlaxElectraForTokenClassification,
     FlaxElectraModel,
 )





class      UpperCamelCase_       (				unittest.TestCase     ):



  def __init__(					self       ,						snake_case__       ,						snake_case__=13       ,						snake_case__=7       ,						snake_case__=True       ,						snake_case__=True       ,						snake_case__=True       ,						snake_case__=True       ,						snake_case__=99       ,						snake_case__=24       ,						snake_case__=32       ,						snake_case__=5       ,						snake_case__=4       ,						snake_case__=37       ,						snake_case__="gelu"       ,						snake_case__=0.1       ,						snake_case__=0.1       ,						snake_case__=5_12       ,						snake_case__=16       ,						snake_case__=2       ,						snake_case__=0.02       ,						snake_case__=4       ,						)	->	int:



         """simple docstring"""





         UpperCAmelCase					=	parent
         UpperCAmelCase					=	batch_size
         UpperCAmelCase					=	seq_length
         UpperCAmelCase					=	is_training
         UpperCAmelCase					=	use_attention_mask
         UpperCAmelCase					=	use_token_type_ids
         UpperCAmelCase					=	use_labels
         UpperCAmelCase					=	vocab_size
         UpperCAmelCase					=	hidden_size
         UpperCAmelCase					=	embedding_size
         UpperCAmelCase					=	num_hidden_layers
         UpperCAmelCase					=	num_attention_heads
         UpperCAmelCase					=	intermediate_size
         UpperCAmelCase					=	hidden_act
         UpperCAmelCase					=	hidden_dropout_prob
         UpperCAmelCase					=	attention_probs_dropout_prob
         UpperCAmelCase					=	max_position_embeddings
         UpperCAmelCase					=	type_vocab_size
         UpperCAmelCase					=	type_sequence_label_size
         UpperCAmelCase					=	initializer_range
         UpperCAmelCase					=	num_choices



  def 						UpperCamelCase_ (					self )	->	Optional[Any]:



         """simple docstring"""





         UpperCAmelCase					=	ids_tensor([self.batch_size, self.seq_length]       ,						self.vocab_size )

         UpperCAmelCase					=	None
         if self.use_attention_mask:
                UpperCAmelCase					=	random_attention_mask([self.batch_size, self.seq_length] )

         UpperCAmelCase					=	None
         if self.use_token_type_ids:
                UpperCAmelCase					=	ids_tensor([self.batch_size, self.seq_length]       ,						self.type_vocab_size )

         UpperCAmelCase					=	ElectraConfig(
             vocab_size=self.vocab_size       ,						hidden_size=self.hidden_size       ,						embedding_size=self.embedding_size       ,						num_hidden_layers=self.num_hidden_layers       ,						num_attention_heads=self.num_attention_heads       ,						intermediate_size=self.intermediate_size       ,						hidden_act=self.hidden_act       ,						hidden_dropout_prob=self.hidden_dropout_prob       ,						attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,						max_position_embeddings=self.max_position_embeddings       ,						type_vocab_size=self.type_vocab_size       ,						initializer_range=self.initializer_range       ,						)

         return config, input_ids, token_type_ids, attention_mask



  def 						UpperCamelCase_ (					self )	->	List[str]:



         """simple docstring"""





         UpperCAmelCase					=	self.prepare_config_and_inputs()
         UpperCAmelCase			,				UpperCAmelCase			,				UpperCAmelCase			,				UpperCAmelCase					=	config_and_inputs
         UpperCAmelCase					=	{"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": attention_mask}
         return config, inputs_dict





@require_flax
class      UpperCamelCase_       (				a_      ,    unittest.TestCase     ):
  _A				:    str =		True

  _A				:    List[Any] =		(
      (
          FlaxElectraModel,
          FlaxElectraForCausalLM,
          FlaxElectraForMaskedLM,
          FlaxElectraForPreTraining,
          FlaxElectraForTokenClassification,
          FlaxElectraForQuestionAnswering,
          FlaxElectraForMultipleChoice,
          FlaxElectraForSequenceClassification,
      )
      if is_flax_available()
      else ()
  )



  def 						UpperCamelCase_ (					self )	->	str:



         """simple docstring"""





         UpperCAmelCase					=	FlaxElectraModelTester(self )



  @slow
  def 						UpperCamelCase_ (					self )	->	Tuple:



         """simple docstring"""





         for model_class_name in self.all_model_classes:
                if model_class_name == FlaxElectraForMaskedLM:
                       UpperCAmelCase					=	model_class_name.from_pretrained("""google/electra-small-generator""" )
                else:
                       UpperCAmelCase					=	model_class_name.from_pretrained("""google/electra-small-discriminator""" )
                UpperCAmelCase					=	model(np.ones((1, 1) ) )
                self.assertIsNotNone(snake_case__ )
