


'''simple docstring'''







import tempfile
import unittest

import numpy as np

import transformers
from transformers import GPTaConfig, GPTaTokenizer, is_flax_available, is_torch_available
from transformers.testing_utils import is_pt_flax_cross_test, require_flax, slow

from ...generation.test_flax_utils import FlaxGenerationTesterMixin
from ...test_modeling_flax_common import FlaxModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask


if is_flax_available():
       import jax
       import jax.numpy as jnp

       from transformers.modeling_flax_pytorch_utils import (
           convert_pytorch_state_dict_to_flax,
           load_flax_weights_in_pytorch_model,
       )
       from transformers.models.gpta.modeling_flax_gpta import FlaxGPTaLMHeadModel, FlaxGPTaModel

if is_torch_available():
       import torch
class 		UpperCamelCase__    :



     """simple docstring"""






     def __init__(       self					,		snake_case__					,		snake_case__=14					,		snake_case__=7					,		snake_case__=True					,		snake_case__=True					,		snake_case__=False					,		snake_case__=True					,		snake_case__=99					,		snake_case__=32					,		snake_case__=5					,		snake_case__=4					,		snake_case__=37					,		snake_case__="gelu"					,		snake_case__=0.1					,		snake_case__=0.1					,		snake_case__=512					,		snake_case__=0.02					,		):

            '''simple docstring'''

            _lowerCAmelCase			:							Union[str, Any]      	=				parent
            _lowerCAmelCase			:							Any      	=				batch_size
            _lowerCAmelCase			:							Optional[int]      	=				seq_length
            _lowerCAmelCase			:							Tuple      	=				is_training
            _lowerCAmelCase			:							Optional[int]      	=				use_input_mask
            _lowerCAmelCase			:							Any      	=				use_token_type_ids
            _lowerCAmelCase			:							List[str]      	=				use_labels
            _lowerCAmelCase			:							List[str]      	=				vocab_size
            _lowerCAmelCase			:							Any      	=				hidden_size
            _lowerCAmelCase			:							Dict      	=				num_hidden_layers
            _lowerCAmelCase			:							str      	=				num_attention_heads
            _lowerCAmelCase			:							List[str]      	=				intermediate_size
            _lowerCAmelCase			:							Optional[Any]      	=				hidden_act
            _lowerCAmelCase			:							Tuple      	=				hidden_dropout_prob
            _lowerCAmelCase			:							str      	=				attention_probs_dropout_prob
            _lowerCAmelCase			:							Tuple      	=				max_position_embeddings
            _lowerCAmelCase			:							str      	=				initializer_range
            _lowerCAmelCase			:							List[str]      	=				None
            _lowerCAmelCase			:							str      	=				vocab_size - 1
            _lowerCAmelCase			:							Any      	=				vocab_size - 1
            _lowerCAmelCase			:							Dict      	=				vocab_size - 1







     def a		(       self      ):

            '''simple docstring'''

            _lowerCAmelCase			:							int      	=				ids_tensor([self.batch_size, self.seq_length]					,		self.vocab_size      )

            _lowerCAmelCase			:							List[Any]      	=				None
            if self.use_input_mask:
                   _lowerCAmelCase			:							int      	=				random_attention_mask([self.batch_size, self.seq_length]      )

            _lowerCAmelCase			:							Optional[Any]      	=				GPTaConfig(
                vocab_size=self.vocab_size					,		n_embd=self.hidden_size					,		n_layer=self.num_hidden_layers					,		n_head=self.num_attention_heads					,		n_positions=self.max_position_embeddings					,		use_cache=__a					,		bos_token_id=self.bos_token_id					,		eos_token_id=self.eos_token_id					,		pad_token_id=self.pad_token_id					,		)

            return (config, input_ids, input_mask)







     def a		(       self      ):

            '''simple docstring'''

            _lowerCAmelCase			:							Optional[Any]      	=				self.prepare_config_and_inputs()
            _lowerCAmelCase			:							Union[str, Any]      	=				config_and_inputs
            _lowerCAmelCase			:							Optional[int]      	=				{"input_ids": input_ids, "attention_mask": attention_mask}
            return config, inputs_dict







     def a		(       self      ):

            '''simple docstring'''

            _lowerCAmelCase			:							int      	=				self.prepare_config_and_inputs()

            _lowerCAmelCase			:							Tuple      	=				floats_tensor([self.batch_size, self.seq_length, self.hidden_size]      )
            _lowerCAmelCase			:							Optional[int]      	=				ids_tensor([self.batch_size, self.seq_length]					,		vocab_size=2      )

            return (
                config,
                input_ids,
                attention_mask,
                encoder_hidden_states,
                encoder_attention_mask,
            )







     def a		(       self					,		snake_case__					,		snake_case__					,		snake_case__					,		snake_case__      ):

            '''simple docstring'''

            _lowerCAmelCase			:							List[str]      	=				20
            _lowerCAmelCase			:							Tuple      	=				model_class_name(__a      )

            _lowerCAmelCase			:							Optional[Any]      	=				model.init_cache(input_ids.shape[0]					,		__a      )
            _lowerCAmelCase			:							Union[str, Any]      	=				jnp.ones((input_ids.shape[0], max_decoder_length)					,		dtype='i4'      )

            _lowerCAmelCase			:							Optional[Any]      	=				jnp.broadcast_to(
                jnp.arange(input_ids.shape[-1] - 1      )[None, :]					,		(input_ids.shape[0], input_ids.shape[-1] - 1)      )
            _lowerCAmelCase			:							Optional[int]      	=				model(
                input_ids[:, :-1]					,		attention_mask=__a					,		past_key_values=__a					,		position_ids=__a					,		)

            _lowerCAmelCase			:							Dict      	=				jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]					,		dtype='i4'      )
            _lowerCAmelCase			:							Optional[Any]      	=				model(
                input_ids[:, -1:]					,		attention_mask=__a					,		past_key_values=outputs_cache.past_key_values					,		position_ids=__a					,		)

            _lowerCAmelCase			:							Optional[Any]      	=				model(__a      )

            _lowerCAmelCase			:							Optional[int]      	=				np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])      )      )
            self.parent.assertTrue(diff < 1E-3					,		msg=F'Max diff is {diff}'      )







     def a		(       self					,		snake_case__					,		snake_case__					,		snake_case__					,		snake_case__      ):

            '''simple docstring'''

            _lowerCAmelCase			:							Optional[Any]      	=				20
            _lowerCAmelCase			:							Tuple      	=				model_class_name(__a      )

            _lowerCAmelCase			:							Any      	=				jnp.concatenate(
                [attention_mask, jnp.zeros((attention_mask.shape[0], max_decoder_length - attention_mask.shape[1])      )]					,		axis=-1					,		)

            _lowerCAmelCase			:							str      	=				model.init_cache(input_ids.shape[0]					,		__a      )
            _lowerCAmelCase			:							str      	=				jnp.broadcast_to(
                jnp.arange(input_ids.shape[-1] - 1      )[None, :]					,		(input_ids.shape[0], input_ids.shape[-1] - 1)      )

            _lowerCAmelCase			:							Tuple      	=				model(
                input_ids[:, :-1]					,		attention_mask=__a					,		past_key_values=__a					,		position_ids=__a					,		)
            _lowerCAmelCase			:							Any      	=				jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]					,		dtype='i4'      )
            _lowerCAmelCase			:							List[Any]      	=				model(
                input_ids[:, -1:]					,		past_key_values=outputs_cache.past_key_values					,		attention_mask=__a					,		position_ids=__a					,		)

            _lowerCAmelCase			:							Tuple      	=				model(__a					,		attention_mask=__a      )

            _lowerCAmelCase			:							int      	=				np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])      )      )
            self.parent.assertTrue(diff < 1E-3					,		msg=F'Max diff is {diff}'      )
@require_flax
class 		UpperCamelCase__    (       _UpperCamelCase			,    _UpperCamelCase			,    unittest.TestCase		):



     """simple docstring"""






     __magic_name__				=			(FlaxGPTaModel, FlaxGPTaLMHeadModel) if is_flax_available() else ()
     __magic_name__				=			(FlaxGPTaLMHeadModel,) if is_flax_available() else ()







     def a		(       self      ):

            '''simple docstring'''

            _lowerCAmelCase			:							str      	=				FlaxGPTaModelTester(self      )







     def a		(       self      ):

            '''simple docstring'''

            for model_class_name in self.all_model_classes:
                   _lowerCAmelCase			:							List[str]      	=				self.model_tester.prepare_config_and_inputs()
                   self.model_tester.check_use_cache_forward(__a					,		__a					,		__a					,		__a      )







     def a		(       self      ):

            '''simple docstring'''

            for model_class_name in self.all_model_classes:
                   _lowerCAmelCase			:							int      	=				self.model_tester.prepare_config_and_inputs()
                   self.model_tester.check_use_cache_forward_with_attn_mask(
                       __a					,		__a					,		__a					,		__a      )







     @slow
     def a		(       self      ):

            '''simple docstring'''

            _lowerCAmelCase			:							Optional[int]      	=				GPTaTokenizer.from_pretrained('gpt2'					,		pad_token='</s>'					,		padding_side='left'      )
            _lowerCAmelCase			:							Optional[int]      	=				tokenizer(['Hello this is a long string', 'Hey']					,		return_tensors='np'					,		padding=__a					,		truncation=__a      )

            _lowerCAmelCase			:							Tuple      	=				FlaxGPTaLMHeadModel.from_pretrained('gpt2'      )
            _lowerCAmelCase			:							Optional[Any]      	=				False
            _lowerCAmelCase			:							str      	=				model.config.eos_token_id

            _lowerCAmelCase			:							Any      	=				jax.jit(model.generate      )

            _lowerCAmelCase			:							Optional[Any]      	=				jit_generate(inputs['input_ids']					,		attention_mask=inputs['attention_mask']      ).sequences

            _lowerCAmelCase			:							Tuple      	=				tokenizer.batch_decode(__a					,		skip_special_tokens=__a      )

            _lowerCAmelCase			:							Any      	=				[
                "Hello this is a long string of words. I'm going to start with the first one.\n",
                "Hey, I'm not sure if I'm going to be able to do",
            ]

            self.assertListEqual(__a					,		__a      )







     @is_pt_flax_cross_test
     def a		(       self      ):

            '''simple docstring'''

            _lowerCAmelCase			:							List[Any]      	=				self.model_tester.prepare_config_and_inputs_for_common()

            for model_class in self.all_model_classes:
                   with self.subTest(model_class.__name__      ):
                          # prepare inputs
                          _lowerCAmelCase			:							Optional[Any]      	=				self._prepare_for_class(__a					,		__a      )
                          _lowerCAmelCase			:							int      	=				{k: torch.tensor(v.tolist()      ) for k, v in prepared_inputs_dict.items()}

                          # load corresponding PyTorch class
                          _lowerCAmelCase			:							str      	=				model_class.__name__[4:]  # Skip the "Flax" at the beginning
                          _lowerCAmelCase			:							Union[str, Any]      	=				getattr(__a					,		__a      )

                          _lowerCAmelCase			:							Optional[Any]      	=				pt_inputs["input_ids"].shape
                          _lowerCAmelCase			:							Tuple      	=				np.random.randint(0					,		seq_length - 1					,		size=(batch_size,)      )
                          for batch_idx, start_index in enumerate(__a      ):
                                 _lowerCAmelCase			:							Optional[int]      	=				0
                                 _lowerCAmelCase			:							List[str]      	=				1
                                 _lowerCAmelCase			:							Dict      	=				0
                                 _lowerCAmelCase			:							Optional[Any]      	=				1
                          _lowerCAmelCase			:							Union[str, Any]      	=				pt_model_class(__a      ).eval()
                          _lowerCAmelCase			:							Any      	=				model_class(__a					,		dtype=jnp.floataa      )

                          _lowerCAmelCase			:							int      	=				convert_pytorch_state_dict_to_flax(pt_model.state_dict()					,		__a      )
                          _lowerCAmelCase			:							List[Any]      	=				fx_state

                          with torch.no_grad():
                                 _lowerCAmelCase			:							List[str]      	=				pt_model(**__a      ).to_tuple()

                          _lowerCAmelCase			:							str      	=				fx_model(**__a      ).to_tuple()
                          self.assertEqual(len(__a      )					,		len(__a      )					,		'Output lengths differ between Flax and PyTorch'      )
                          for fx_output, pt_output in zip(__a					,		__a      ):
                                 self.assert_almost_equals(fx_output[:, -1]					,		pt_output[:, -1].numpy()					,		4E-2      )

                          with tempfile.TemporaryDirectory() as tmpdirname:
                                 pt_model.save_pretrained(__a      )
                                 _lowerCAmelCase			:							Any      	=				model_class.from_pretrained(__a					,		from_pt=__a      )

                          _lowerCAmelCase			:							Tuple      	=				fx_model_loaded(**__a      ).to_tuple()
                          self.assertEqual(
                              len(__a      )					,		len(__a      )					,		'Output lengths differ between Flax and PyTorch'      )
                          for fx_output_loaded, pt_output in zip(__a					,		__a      ):
                                 self.assert_almost_equals(fx_output_loaded[:, -1]					,		pt_output[:, -1].numpy()					,		4E-2      )







     @is_pt_flax_cross_test
     def a		(       self      ):

            '''simple docstring'''

            _lowerCAmelCase			:							Optional[Any]      	=				self.model_tester.prepare_config_and_inputs_for_common()
            for model_class in self.all_model_classes:
                   with self.subTest(model_class.__name__      ):
                          # prepare inputs
                          _lowerCAmelCase			:							Union[str, Any]      	=				self._prepare_for_class(__a					,		__a      )
                          _lowerCAmelCase			:							Optional[Any]      	=				{k: torch.tensor(v.tolist()      ) for k, v in prepared_inputs_dict.items()}

                          # load corresponding PyTorch class
                          _lowerCAmelCase			:							Any      	=				model_class.__name__[4:]  # Skip the "Flax" at the beginning
                          _lowerCAmelCase			:							Union[str, Any]      	=				getattr(__a					,		__a      )

                          _lowerCAmelCase			:							Dict      	=				pt_model_class(__a      ).eval()
                          _lowerCAmelCase			:							Tuple      	=				model_class(__a					,		dtype=jnp.floataa      )

                          _lowerCAmelCase			:							Dict      	=				load_flax_weights_in_pytorch_model(__a					,		fx_model.params      )
                          _lowerCAmelCase			:							int      	=				pt_inputs["input_ids"].shape
                          _lowerCAmelCase			:							Optional[Any]      	=				np.random.randint(0					,		seq_length - 1					,		size=(batch_size,)      )
                          for batch_idx, start_index in enumerate(__a      ):
                                 _lowerCAmelCase			:							Dict      	=				0
                                 _lowerCAmelCase			:							Union[str, Any]      	=				1
                                 _lowerCAmelCase			:							Dict      	=				0
                                 _lowerCAmelCase			:							Optional[int]      	=				1

                          # make sure weights are tied in PyTorch
                          pt_model.tie_weights()

                          with torch.no_grad():
                                 _lowerCAmelCase			:							Optional[int]      	=				pt_model(**__a      ).to_tuple()

                          _lowerCAmelCase			:							List[Any]      	=				fx_model(**__a      ).to_tuple()
                          self.assertEqual(len(__a      )					,		len(__a      )					,		'Output lengths differ between Flax and PyTorch'      )
                          for fx_output, pt_output in zip(__a					,		__a      ):
                                 self.assert_almost_equals(fx_output[:, -1]					,		pt_output[:, -1].numpy()					,		4E-2      )

                          with tempfile.TemporaryDirectory() as tmpdirname:
                                 fx_model.save_pretrained(__a      )
                                 _lowerCAmelCase			:							Tuple      	=				pt_model_class.from_pretrained(__a					,		from_flax=__a      )

                          with torch.no_grad():
                                 _lowerCAmelCase			:							int      	=				pt_model_loaded(**__a      ).to_tuple()

                          self.assertEqual(
                              len(__a      )					,		len(__a      )					,		'Output lengths differ between Flax and PyTorch'      )
                          for fx_output, pt_output in zip(__a					,		__a      ):
                                 self.assert_almost_equals(fx_output[:, -1]					,		pt_output[:, -1].numpy()					,		4E-2      )







     @slow
     def a		(       self      ):

            '''simple docstring'''

            for model_class_name in self.all_model_classes:
                   _lowerCAmelCase			:							Union[str, Any]      	=				model_class_name.from_pretrained('gpt2'					,		from_pt=__a      )
                   _lowerCAmelCase			:							List[str]      	=				model(np.ones((1, 1)      )      )
                   self.assertIsNotNone(__a      )








"""simple docstring"""


from __future__ import annotations




def     lowercase__			(    lowercase_						,lowercase_						,lowercase_	)    ->						int:




      """simple docstring"""

      assert n > 1 and a > 0 and greatest_common_divisor(lowercase_						,lowercase_	) == 1
      ((_UpperCamelCase),     (_UpperCamelCase),     (_UpperCamelCase))					:		int  =				extended_gcd(lowercase_						,lowercase_	)  # Implemented below
      _UpperCamelCase					:		Any  =				(b * s) % n
      return x




def     lowercase__			(    lowercase_						,lowercase_	)    ->						int:




      """simple docstring"""

      ((_UpperCamelCase),     (_UpperCamelCase))					:		List[Any]  =				extended_euclid(lowercase_						,lowercase_	)  # Implemented below
      if b < 0:
            _UpperCamelCase					:		Any  =				(b % n + n) % n
      return b




def     lowercase__			(    lowercase_						,lowercase_						,lowercase_	)    ->						int:




      """simple docstring"""

      _UpperCamelCase					:		Union[str, Any]  =				invert_modulo(lowercase_						,lowercase_	)
      _UpperCamelCase					:		Any  =				(b * s) % n
      return x




def     lowercase__			(    lowercase_						,lowercase_	)    ->						tuple[int, int, int]:




      """simple docstring"""

      assert a >= 0 and b >= 0

      if b == 0:
            _UpperCamelCase,     _UpperCamelCase,     _UpperCamelCase					:		str  =				a, 1, 0
      else:
            ((_UpperCamelCase),     (_UpperCamelCase),     (_UpperCamelCase))					:		str  =				extended_gcd(lowercase_						,a % b	)
            _UpperCamelCase					:		Dict  =				q
            _UpperCamelCase					:		Tuple  =				p - q * (a // b)

      assert a % d == 0 and b % d == 0
      assert d == a * x + b * y

      return (d, x, y)




def     lowercase__			(    lowercase_						,lowercase_	)    ->						tuple[int, int]:




      """simple docstring"""

      if b == 0:
            return (1, 0)
      ((_UpperCamelCase),     (_UpperCamelCase))					:		Optional[Any]  =				extended_euclid(lowercase_						,a % b	)
      _UpperCamelCase					:		Dict  =				a // b
      return (y, x - k * y)




def     lowercase__			(    lowercase_						,lowercase_	)    ->						int:




      """simple docstring"""

      if a < b:
            _UpperCamelCase,     _UpperCamelCase					:		List[Any]  =				b, a

      while a % b != 0:
            _UpperCamelCase,     _UpperCamelCase					:		List[Any]  =				b, a % b

      return b


if __name__ == "__main__":
  from doctest import testmod

  testmod(name="modular_division", verbose=True)
  testmod(name="modular_division2", verbose=True)
  testmod(name="invert_modulo", verbose=True)
  testmod(name="extended_gcd", verbose=True)
  testmod(name="extended_euclid", verbose=True)
  testmod(name="greatest_common_divisor", verbose=True)
