
"""simple docstring"""


from ...configuration_utils import PretrainedConfig
from ...utils import logging


_lowercase				 =					logging.get_logger(__name__)

_lowercase				 =					{
    's-JoL/Open-Llama-V1': 'https://huggingface.co/s-JoL/Open-Llama-V1/blob/main/config.json',
}







class 					lowerCAmelCase_					(   _lowercase     ):



						'''simple docstring'''


						_lowerCamelCase:     Union[str, Any]   =     'open-llama'


						def __init__(			self  :   Tuple	,A_  :   Optional[int]=10_0000	,A_  :   Tuple=4096	,A_  :   Any=1_1008	,A_  :   List[Any]=32	,A_  :   Union[str, Any]=32	,A_  :   Any="silu"	,A_  :   int=2048	,A_  :   List[Any]=0.02	,A_  :   Optional[Any]=1e-6	,A_  :   List[Any]=True	,A_  :   List[Any]=0	,A_  :   List[str]=1	,A_  :   Dict=2	,A_  :   Optional[Any]=False	,A_  :   int=True	,A_  :   Any=0.1	,A_  :   Optional[int]=0.1	,A_  :   int=True	,A_  :   Union[str, Any]=True	,A_  :   Dict=None	,**A_  :   Any	,)  -> Optional[int]:
							A     = vocab_size
							A     = max_position_embeddings
							A     = hidden_size
							A     = intermediate_size
							A     = num_hidden_layers
							A     = num_attention_heads
							A     = hidden_act
							A     = initializer_range
							A     = rms_norm_eps
							A     = use_cache
							A     = kwargs.pop(
							    'use_memorry_efficient_attention'	,lowerCAmelCase__   )
							A     = hidden_dropout_prob
							A     = attention_dropout_prob
							A     = use_stable_embedding
							A     = shared_input_output_embedding
							A     = rope_scaling
							self._rope_scaling_validation()

							super().__init__(
							    pad_token_id=lowerCAmelCase__	,bos_token_id=lowerCAmelCase__	,eos_token_id=lowerCAmelCase__	,tie_word_embeddings=lowerCAmelCase__	,**lowerCAmelCase__	,)




						def     _SCREAMING_SNAKE_CASE   (			self  :   Optional[Any]   )  -> Optional[int]:




							if self.rope_scaling is None:
								return

							if not isinstance(self.rope_scaling	,lowerCAmelCase__   ) or len(self.rope_scaling   ) != 2:
								raise ValueError(
								    '`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '
								    F'got {self.rope_scaling}'   )
							A     = self.rope_scaling.get('type'	,lowerCAmelCase__   )
							A     = self.rope_scaling.get('factor'	,lowerCAmelCase__   )
							if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
								raise ValueError(
								    F'`rope_scaling`\'s name field must be one of [\'linear\', \'dynamic\'], got {rope_scaling_type}'   )
							if rope_scaling_factor is None or not isinstance(lowerCAmelCase__	,lowerCAmelCase__   ) or rope_scaling_factor <= 1.0:
								raise ValueError(F'`rope_scaling`\'s factor field must be an float > 1, got {rope_scaling_factor}'   )






'''simple docstring'''

import json
import os
import unittest

from transformers import OpenAIGPTTokenizer, OpenAIGPTTokenizerFast
from transformers.models.openai.tokenization_openai import VOCAB_FILES_NAMES
from transformers.testing_utils import require_ftfy, require_spacy, require_tokenizers

from ...test_tokenization_common import TokenizerTesterMixin
@require_tokenizers
class 							__lowerCAmelCase	(       __magic_name__		,		unittest.TestCase					):






       """simple docstring"""
       _snake_case      :	int									=       OpenAIGPTTokenizer
       _snake_case      :	Tuple									=       OpenAIGPTTokenizerFast
       _snake_case      :	Union[str, Any]									=       True
       _snake_case      :	Optional[int]									=       False





       def snake_case__		(				self  :						Dict	)       ->	List[Any]:

            '''simple docstring'''




            super().setUp()

            # Adapted from Sennrich et al. 2015 and https://github.com/rsennrich/subword-nmt
            _UpperCamelCase							       =			[
                '''l''',
                '''o''',
                '''w''',
                '''e''',
                '''r''',
                '''s''',
                '''t''',
                '''i''',
                '''d''',
                '''n''',
                '''w</w>''',
                '''r</w>''',
                '''t</w>''',
                '''lo''',
                '''low''',
                '''er</w>''',
                '''low</w>''',
                '''lowest</w>''',
                '''newer</w>''',
                '''wider</w>''',
                '''<unk>''',
            ]
            _UpperCamelCase							       =			dict(zip(lowerCAmelCase__			,					range(len(lowerCAmelCase__	)	)	)	)
            _UpperCamelCase							       =			['''#version: 0.2''', '''l o''', '''lo w''', '''e r</w>''', '''''']

            _UpperCamelCase							       =			os.path.join(self.tmpdirname			,					VOCAB_FILES_NAMES['''vocab_file''']	)
            _UpperCamelCase							       =			os.path.join(self.tmpdirname			,					VOCAB_FILES_NAMES['''merges_file''']	)
            with open(self.vocab_file			,					'''w'''	) as fp:
                 fp.write(json.dumps(lowerCAmelCase__	)	)
            with open(self.merges_file			,					'''w'''	) as fp:
                 fp.write('''\n'''.join(lowerCAmelCase__	)	)





       def snake_case__		(				self  :						Tuple			,					lowerCAmelCase__  :						List[str]	)       ->	Optional[Any]:

            '''simple docstring'''




            return "lower newer", "lower newer"





       def snake_case__		(				self  :						Tuple	)       ->	Union[str, Any]:

            '''simple docstring'''




            _UpperCamelCase							       =			OpenAIGPTTokenizer(self.vocab_file			,					self.merges_file	)

            _UpperCamelCase							       =			'''lower'''
            _UpperCamelCase							       =			['''low''', '''er</w>''']
            _UpperCamelCase							       =			tokenizer.tokenize(lowerCAmelCase__	)
            self.assertListEqual(lowerCAmelCase__			,					lowerCAmelCase__	)

            _UpperCamelCase							       =			tokens + ['''<unk>''']
            _UpperCamelCase							       =			[14, 15, 20]
            self.assertListEqual(tokenizer.convert_tokens_to_ids(lowerCAmelCase__	)			,					lowerCAmelCase__	)





       def snake_case__		(				self  :						Union[str, Any]			,					lowerCAmelCase__  :						Tuple=15	)       ->	Union[str, Any]:

            '''simple docstring'''




            for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
                 with self.subTest(f"""{tokenizer.__class__.__name__} ({pretrained_name})"""	):
                      _UpperCamelCase							       =			self.rust_tokenizer_class.from_pretrained(lowerCAmelCase__			,					**lowerCAmelCase__	)

                      # Simple input
                      _UpperCamelCase							       =			'''This is a simple input'''
                      _UpperCamelCase							       =			['''This is a simple input 1''', '''This is a simple input 2''']
                      _UpperCamelCase							       =			('''This is a simple input''', '''This is a pair''')
                      _UpperCamelCase							       =			[
                          ('''This is a simple input 1''', '''This is a simple input 2'''),
                          ('''This is a simple pair 1''', '''This is a simple pair 2'''),
                      ]

                      # Simple input tests
                      self.assertRaises(lowerCAmelCase__			,					tokenizer_r.encode			,					lowerCAmelCase__			,					max_length=lowerCAmelCase__			,					padding='''max_length'''	)

                      # Simple input
                      self.assertRaises(lowerCAmelCase__			,					tokenizer_r.encode_plus			,					lowerCAmelCase__			,					max_length=lowerCAmelCase__			,					padding='''max_length'''	)

                      # Simple input
                      self.assertRaises(
                          lowerCAmelCase__			,					tokenizer_r.batch_encode_plus			,					lowerCAmelCase__			,					max_length=lowerCAmelCase__			,					padding='''max_length'''			,					)

                      # Pair input
                      self.assertRaises(lowerCAmelCase__			,					tokenizer_r.encode			,					lowerCAmelCase__			,					max_length=lowerCAmelCase__			,					padding='''max_length'''	)

                      # Pair input
                      self.assertRaises(lowerCAmelCase__			,					tokenizer_r.encode_plus			,					lowerCAmelCase__			,					max_length=lowerCAmelCase__			,					padding='''max_length'''	)

                      # Pair input
                      self.assertRaises(
                          lowerCAmelCase__			,					tokenizer_r.batch_encode_plus			,					lowerCAmelCase__			,					max_length=lowerCAmelCase__			,					padding='''max_length'''			,					)







       def snake_case__		(				self  :						Tuple	)       ->	Optional[Any]:

            '''simple docstring'''




            pass

@require_ftfy
@require_spacy
@require_tokenizers
class 							__lowerCAmelCase	(       __magic_name__					):






       """simple docstring"""
       pass



