


'''simple docstring'''






from ...configuration_utils import PretrainedConfig
from ...utils import logging


__UpperCAmelCase					       =							logging.get_logger(__name__)

__UpperCAmelCase					       =							{
    '''s-JoL/Open-Llama-V1''': '''https://huggingface.co/s-JoL/Open-Llama-V1/blob/main/config.json''',
}




class 	a__       (							a__						):





       '''simple docstring'''






       lowercase__    :     Union[str, Any]					=   "open-llama"







       def __init__(	self      ,					lowerCamelCase_=10_00_00      ,					lowerCamelCase_=40_96      ,					lowerCamelCase_=1_10_08      ,					lowerCamelCase_=32      ,					lowerCamelCase_=32      ,					lowerCamelCase_="silu"      ,					lowerCamelCase_=20_48      ,					lowerCamelCase_=0.02      ,					lowerCamelCase_=1e-6      ,					lowerCamelCase_=True      ,					lowerCamelCase_=0      ,					lowerCamelCase_=1      ,					lowerCamelCase_=2      ,					lowerCamelCase_=False      ,					lowerCamelCase_=True      ,					lowerCamelCase_=0.1      ,					lowerCamelCase_=0.1      ,					lowerCamelCase_=True      ,					lowerCamelCase_=True      ,					lowerCamelCase_=None      ,					**lowerCamelCase_      ,					) ->							List[Any]:
              lowerCAmelCase__      =			vocab_size
              lowerCAmelCase__      =			max_position_embeddings
              lowerCAmelCase__      =			hidden_size
              lowerCAmelCase__      =			intermediate_size
              lowerCAmelCase__      =			num_hidden_layers
              lowerCAmelCase__      =			num_attention_heads
              lowerCAmelCase__      =			hidden_act
              lowerCAmelCase__      =			initializer_range
              lowerCAmelCase__      =			rms_norm_eps
              lowerCAmelCase__      =			use_cache
              lowerCAmelCase__      =			kwargs.pop(
                  '''use_memorry_efficient_attention'''      ,					lowerCamelCase_     )
              lowerCAmelCase__      =			hidden_dropout_prob
              lowerCAmelCase__      =			attention_dropout_prob
              lowerCAmelCase__      =			use_stable_embedding
              lowerCAmelCase__      =			shared_input_output_embedding
              lowerCAmelCase__      =			rope_scaling
              self._rope_scaling_validation()

              super().__init__(
                  pad_token_id=lowerCamelCase_      ,					bos_token_id=lowerCamelCase_      ,					eos_token_id=lowerCamelCase_      ,					tie_word_embeddings=lowerCamelCase_      ,					**lowerCamelCase_      ,					)







       def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							Optional[Any]:
              if self.rope_scaling is None:
                     return

              if not isinstance(self.rope_scaling      ,					lowerCamelCase_     ) or len(self.rope_scaling     ) != 2:
                     raise ValueError(
                         '''`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '''
                         F"""got {self.rope_scaling}"""     )
              lowerCAmelCase__      =			self.rope_scaling.get('''type'''      ,					lowerCamelCase_     )
              lowerCAmelCase__      =			self.rope_scaling.get('''factor'''      ,					lowerCamelCase_     )
              if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
                     raise ValueError(
                         F"""`rope_scaling`'s name field must be one of ['linear', 'dynamic'], got {rope_scaling_type}"""     )
              if rope_scaling_factor is None or not isinstance(lowerCamelCase_      ,					lowerCamelCase_     ) or rope_scaling_factor <= 1.0:
                     raise ValueError(F"""`rope_scaling`'s factor field must be an float > 1, got {rope_scaling_factor}"""     )



'''simple docstring'''






import tempfile
import unittest

import numpy as np
from huggingface_hub import HfFolder, delete_repo
from requests.exceptions import HTTPError

from transformers import BertConfig, is_flax_available
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_flax


if is_flax_available():
		import os

		from flax.core.frozen_dict import unfreeze
		from flax.traverse_util import flatten_dict

		from transformers import FlaxBertModel

		__UpperCAmelCase					       =							'''0.12'''  # assumed parallelism: 8




@require_flax
@is_staging_test
class 	a__       (							unittest.TestCase						):





							'''simple docstring'''






							@classmethod
							def 						__SCREAMING_SNAKE_CASE							(	cls     ) ->							Any:
														lowerCAmelCase__      =			TOKEN
														HfFolder.save_token(lowerCamelCase_     )







							@classmethod
							def 						__SCREAMING_SNAKE_CASE							(	cls     ) ->							Tuple:
														try:
																					delete_repo(token=cls._token      ,					repo_id='''test-model-flax'''     )
														except HTTPError:
																					pass

														try:
																					delete_repo(token=cls._token      ,					repo_id='''valid_org/test-model-flax-org'''     )
														except HTTPError:
																					pass







							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							List[Any]:
														lowerCAmelCase__      =			BertConfig(
														    vocab_size=99      ,					hidden_size=32      ,					num_hidden_layers=5      ,					num_attention_heads=4      ,					intermediate_size=37     )
														lowerCAmelCase__      =			FlaxBertModel(lowerCamelCase_     )
														model.push_to_hub('''test-model-flax'''      ,					use_auth_token=self._token     )

														lowerCAmelCase__      =			FlaxBertModel.from_pretrained(F"""{USER}/test-model-flax"""     )

														lowerCAmelCase__      =			flatten_dict(unfreeze(model.params     )     )
														lowerCAmelCase__      =			flatten_dict(unfreeze(new_model.params     )     )

														for key in base_params.keys():
																					lowerCAmelCase__      =			(base_params[key] - new_params[key]).sum().item()
																					self.assertLessEqual(lowerCamelCase_      ,					1e-3      ,					msg=F"""{key} not identical"""     )

														# Reset repo
														delete_repo(token=self._token      ,					repo_id='''test-model-flax'''     )

														# Push to hub via save_pretrained
														with tempfile.TemporaryDirectory() as tmp_dir:
																					model.save_pretrained(lowerCamelCase_      ,					repo_id='''test-model-flax'''      ,					push_to_hub=lowerCamelCase_      ,					use_auth_token=self._token     )

														lowerCAmelCase__      =			FlaxBertModel.from_pretrained(F"""{USER}/test-model-flax"""     )

														lowerCAmelCase__      =			flatten_dict(unfreeze(model.params     )     )
														lowerCAmelCase__      =			flatten_dict(unfreeze(new_model.params     )     )

														for key in base_params.keys():
																					lowerCAmelCase__      =			(base_params[key] - new_params[key]).sum().item()
																					self.assertLessEqual(lowerCamelCase_      ,					1e-3      ,					msg=F"""{key} not identical"""     )







							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							Union[str, Any]:
														lowerCAmelCase__      =			BertConfig(
														    vocab_size=99      ,					hidden_size=32      ,					num_hidden_layers=5      ,					num_attention_heads=4      ,					intermediate_size=37     )
														lowerCAmelCase__      =			FlaxBertModel(lowerCamelCase_     )
														model.push_to_hub('''valid_org/test-model-flax-org'''      ,					use_auth_token=self._token     )

														lowerCAmelCase__      =			FlaxBertModel.from_pretrained('''valid_org/test-model-flax-org'''     )

														lowerCAmelCase__      =			flatten_dict(unfreeze(model.params     )     )
														lowerCAmelCase__      =			flatten_dict(unfreeze(new_model.params     )     )

														for key in base_params.keys():
																					lowerCAmelCase__      =			(base_params[key] - new_params[key]).sum().item()
																					self.assertLessEqual(lowerCamelCase_      ,					1e-3      ,					msg=F"""{key} not identical"""     )

														# Reset repo
														delete_repo(token=self._token      ,					repo_id='''valid_org/test-model-flax-org'''     )

														# Push to hub via save_pretrained
														with tempfile.TemporaryDirectory() as tmp_dir:
																					model.save_pretrained(
																					    lowerCamelCase_      ,					repo_id='''valid_org/test-model-flax-org'''      ,					push_to_hub=lowerCamelCase_      ,					use_auth_token=self._token     )

														lowerCAmelCase__      =			FlaxBertModel.from_pretrained('''valid_org/test-model-flax-org'''     )

														lowerCAmelCase__      =			flatten_dict(unfreeze(model.params     )     )
														lowerCAmelCase__      =			flatten_dict(unfreeze(new_model.params     )     )

														for key in base_params.keys():
																					lowerCAmelCase__      =			(base_params[key] - new_params[key]).sum().item()
																					self.assertLessEqual(lowerCamelCase_      ,					1e-3      ,					msg=F"""{key} not identical"""     )


def       _snake_case     (  A				,					A )		->					Optional[int]:
							lowerCAmelCase__      =			True
							lowerCAmelCase__      =			flatten_dict(modela.params )
							lowerCAmelCase__      =			flatten_dict(modela.params )
							for key in flat_params_a.keys():
														if np.sum(np.abs(flat_params_a[key] - flat_params_a[key] ) ) > 1E-4:
																					lowerCAmelCase__      =			False

							return models_are_equal






@require_flax
class 	a__       (							unittest.TestCase						):





							'''simple docstring'''






							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							Optional[Any]:
														lowerCAmelCase__      =			BertConfig.from_pretrained('''hf-internal-testing/tiny-bert-flax-only'''     )
														lowerCAmelCase__      =			FlaxBertModel(lowerCamelCase_     )

														lowerCAmelCase__      =			'''bert'''
														with tempfile.TemporaryDirectory() as tmp_dir:
																					model.save_pretrained(os.path.join(lowerCamelCase_      ,					lowerCamelCase_     )     )

																					with self.assertRaises(lowerCamelCase_     ):
																												lowerCAmelCase__      =			FlaxBertModel.from_pretrained(lowerCamelCase_     )

																					lowerCAmelCase__      =			FlaxBertModel.from_pretrained(lowerCamelCase_      ,					subfolder=lowerCamelCase_     )

														self.assertTrue(check_models_equal(lowerCamelCase_      ,					lowerCamelCase_     )     )







							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							List[Any]:
														lowerCAmelCase__      =			BertConfig.from_pretrained('''hf-internal-testing/tiny-bert-flax-only'''     )
														lowerCAmelCase__      =			FlaxBertModel(lowerCamelCase_     )

														lowerCAmelCase__      =			'''bert'''
														with tempfile.TemporaryDirectory() as tmp_dir:
																					model.save_pretrained(os.path.join(lowerCamelCase_      ,					lowerCamelCase_     )      ,					max_shard_size='''10KB'''     )

																					with self.assertRaises(lowerCamelCase_     ):
																												lowerCAmelCase__      =			FlaxBertModel.from_pretrained(lowerCamelCase_     )

																					lowerCAmelCase__      =			FlaxBertModel.from_pretrained(lowerCamelCase_      ,					subfolder=lowerCamelCase_     )

														self.assertTrue(check_models_equal(lowerCamelCase_      ,					lowerCamelCase_     )     )







							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							Optional[Any]:
														lowerCAmelCase__      =			'''bert'''
														lowerCAmelCase__      =			'''hf-internal-testing/tiny-random-bert-subfolder'''

														with self.assertRaises(lowerCamelCase_     ):
																					lowerCAmelCase__      =			FlaxBertModel.from_pretrained(lowerCamelCase_     )

														lowerCAmelCase__      =			FlaxBertModel.from_pretrained(lowerCamelCase_      ,					subfolder=lowerCamelCase_     )

														self.assertIsNotNone(lowerCamelCase_     )







							def 						__SCREAMING_SNAKE_CASE							(	self     ) ->							List[Any]:
														lowerCAmelCase__      =			'''bert'''
														lowerCAmelCase__      =			'''hf-internal-testing/tiny-random-bert-sharded-subfolder'''
														with self.assertRaises(lowerCamelCase_     ):
																					lowerCAmelCase__      =			FlaxBertModel.from_pretrained(lowerCamelCase_     )

														lowerCAmelCase__      =			FlaxBertModel.from_pretrained(lowerCamelCase_      ,					subfolder=lowerCamelCase_     )

														self.assertIsNotNone(lowerCamelCase_     )