



"""simple docstring"""


from ...configuration_utils import PretrainedConfig
from ...utils import logging


__A										=   logging.get_logger(__name__)

__A										=   {
    "s-JoL/Open-Llama-V1": "https://huggingface.co/s-JoL/Open-Llama-V1/blob/main/config.json",
}
class 			_a (					__lowerCamelCase):
			"""simple docstring"""
			UpperCamelCase__			      =	"""open-llama"""



			def __init__(		self     :    Optional[int]				,   __UpperCamelCase     :    Any=1_0_0_0_0_0				,   __UpperCamelCase     :    Union[str, Any]=4_0_9_6				,   __UpperCamelCase     :    int=1_1_0_0_8				,   __UpperCamelCase     :    Dict=3_2				,   __UpperCamelCase     :    List[Any]=3_2				,   __UpperCamelCase     :    Optional[int]="silu"				,   __UpperCamelCase     :    Optional[int]=2_0_4_8				,   __UpperCamelCase     :    str=0.0_2				,   __UpperCamelCase     :    Optional[Any]=1e-6				,   __UpperCamelCase     :    int=True				,   __UpperCamelCase     :    Any=0				,   __UpperCamelCase     :    Optional[int]=1				,   __UpperCamelCase     :    Union[str, Any]=2				,   __UpperCamelCase     :    Any=False				,   __UpperCamelCase     :    List[Any]=True				,   __UpperCamelCase     :    Tuple=0.1				,   __UpperCamelCase     :    Union[str, Any]=0.1				,   __UpperCamelCase     :    Tuple=True				,   __UpperCamelCase     :    str=True				,   __UpperCamelCase     :    List[Any]=None				,   **__UpperCamelCase     :    Dict				,   )->str:
							_UpperCAmelCase					=    vocab_size
							_UpperCAmelCase					=    max_position_embeddings
							_UpperCAmelCase					=    hidden_size
							_UpperCAmelCase					=    intermediate_size
							_UpperCAmelCase					=    num_hidden_layers
							_UpperCAmelCase					=    num_attention_heads
							_UpperCAmelCase					=    hidden_act
							_UpperCAmelCase					=    initializer_range
							_UpperCAmelCase					=    rms_norm_eps
							_UpperCAmelCase					=    use_cache
							_UpperCAmelCase					=    kwargs.pop(
							    '''use_memorry_efficient_attention'''				,   UpperCAmelCase_   )
							_UpperCAmelCase					=    hidden_dropout_prob
							_UpperCAmelCase					=    attention_dropout_prob
							_UpperCAmelCase					=    use_stable_embedding
							_UpperCAmelCase					=    shared_input_output_embedding
							_UpperCAmelCase					=    rope_scaling
							self._rope_scaling_validation()

							super().__init__(
							    pad_token_id=UpperCAmelCase_				,   bos_token_id=UpperCAmelCase_				,   eos_token_id=UpperCAmelCase_				,   tie_word_embeddings=UpperCAmelCase_				,   **UpperCAmelCase_				,   )



			def  lowercase__						(		self     :    List[str]   )->Optional[Any]:
							if self.rope_scaling is None:
											return

							if not isinstance(self.rope_scaling				,   UpperCAmelCase_   ) or len(self.rope_scaling   ) != 2:
											raise ValueError(
											    '''`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '''
											    F'got {self.rope_scaling}'   )
							_UpperCAmelCase					=    self.rope_scaling.get('''type'''				,   UpperCAmelCase_   )
							_UpperCAmelCase					=    self.rope_scaling.get('''factor'''				,   UpperCAmelCase_   )
							if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
											raise ValueError(
											    F'`rope_scaling`\'s name field must be one of [\'linear\', \'dynamic\'], got {rope_scaling_type}'   )
							if rope_scaling_factor is None or not isinstance(UpperCAmelCase_				,   UpperCAmelCase_   ) or rope_scaling_factor <= 1.0:
											raise ValueError(F'`rope_scaling`\'s factor field must be an float > 1, got {rope_scaling_factor}'   )








"""simple docstring"""


import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
					from .tokenization_fnet import FNetTokenizer
else:
					__A   :							Tuple										=   None

__A   :							Optional[Any]										=   logging.get_logger(__name__)
__A   :							Union[str, Any]										=   {"vocab_file": "spiece.model", "tokenizer_file": "tokenizer.json"}

__A   :							int										=   {
    "vocab_file": {
        "google/fnet-base": "https://huggingface.co/google/fnet-base/resolve/main/spiece.model",
        "google/fnet-large": "https://huggingface.co/google/fnet-large/resolve/main/spiece.model",
    },
    "tokenizer_file": {
        "google/fnet-base": "https://huggingface.co/google/fnet-base/resolve/main/tokenizer.json",
        "google/fnet-large": "https://huggingface.co/google/fnet-large/resolve/main/tokenizer.json",
    },
}

__A   :							Optional[int]										=   {
    "google/fnet-base": 512,
    "google/fnet-large": 512,
}

__A   :							List[Any]										=   "▁"
class 			_a (					lowerCAmelCase):
			"""simple docstring"""
			UpperCamelCase__			      =	VOCAB_FILES_NAMES
			UpperCamelCase__			      =	PRETRAINED_VOCAB_FILES_MAP
			UpperCamelCase__			      =	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			UpperCamelCase__			      =	["""input_ids""", """token_type_ids"""]
			UpperCamelCase__			      =	FNetTokenizer



			def __init__(		self     :    Optional[Any]				,   __UpperCamelCase     :    Tuple=None				,   __UpperCamelCase     :    Dict=None				,   __UpperCamelCase     :    List[str]=False				,   __UpperCamelCase     :    Union[str, Any]=True				,   __UpperCamelCase     :    List[str]=True				,   __UpperCamelCase     :    List[Any]="<unk>"				,   __UpperCamelCase     :    Any="[SEP]"				,   __UpperCamelCase     :    Dict="<pad>"				,   __UpperCamelCase     :    Optional[int]="[CLS]"				,   __UpperCamelCase     :    List[Any]="[MASK]"				,   **__UpperCamelCase     :    Optional[Any]				,   )->Dict:
							# Mask token behave like a normal word, i.e. include the space before it and
							# is included in the raw text, there should be a match in a non-normalized sentence.
							_UpperCAmelCase					=    (
							    AddedToken(__UpperCamelCase				,   lstrip=__UpperCamelCase				,   rstrip=__UpperCamelCase				,   normalized=__UpperCamelCase   )
							    if isinstance(__UpperCamelCase				,   __UpperCamelCase   )
							    else mask_token
							)

							super().__init__(
							    __UpperCamelCase				,   tokenizer_file=__UpperCamelCase				,   do_lower_case=__UpperCamelCase				,   remove_space=__UpperCamelCase				,   keep_accents=__UpperCamelCase				,   unk_token=__UpperCamelCase				,   sep_token=__UpperCamelCase				,   pad_token=__UpperCamelCase				,   cls_token=__UpperCamelCase				,   mask_token=__UpperCamelCase				,   **__UpperCamelCase				,   )

							_UpperCAmelCase					=    do_lower_case
							_UpperCAmelCase					=    remove_space
							_UpperCAmelCase					=    keep_accents
							_UpperCAmelCase					=    vocab_file
							_UpperCAmelCase					=    False if not self.vocab_file else True



			def  lowercase__						(		self     :    int				,   __UpperCamelCase     :    List[int]				,   __UpperCamelCase     :    Optional[List[int]] = None   )->List[int]:
							_UpperCAmelCase					=    [self.sep_token_id]
							_UpperCAmelCase					=    [self.cls_token_id]
							if token_ids_a is None:
											return cls + token_ids_a + sep
							return cls + token_ids_a + sep + token_ids_a + sep



			def  lowercase__						(		self     :    Optional[Any]				,   __UpperCamelCase     :    List[int]				,   __UpperCamelCase     :    Optional[List[int]] = None   )->List[int]:
							_UpperCAmelCase					=    [self.sep_token_id]
							_UpperCAmelCase					=    [self.cls_token_id]

							if token_ids_a is None:
											return len(cls + token_ids_a + sep   ) * [0]
							return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]



			def  lowercase__						(		self     :    List[str]				,   __UpperCamelCase     :    str				,   __UpperCamelCase     :    Optional[str] = None   )->Tuple[str]:
							if not os.path.isdir(__UpperCamelCase   ):
											logger.error(F'Vocabulary path ({save_directory}) should be a directory'   )
											return
							_UpperCAmelCase					=    os.path.join(
							    __UpperCamelCase				,   (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']   )

							if os.path.abspath(self.vocab_file   ) != os.path.abspath(__UpperCamelCase   ):
											copyfile(self.vocab_file				,   __UpperCamelCase   )

							return (out_vocab_file,)



