
'''simple docstring'''







import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging


if TYPE_CHECKING:
							from transformers.pipelines.conversational import Conversation


_lowercase			:  Optional[Any]   							=	logging.get_logger(__name__)

_lowercase			:  Tuple   							=	{"tokenizer_file": "tokenizer.json"}

_lowercase			:  Dict   							=	{
    "tokenizer_file": {
        "bigscience/tokenizer": "https://huggingface.co/bigscience/tokenizer/blob/main/tokenizer.json",
        "bigscience/bloom-560m": "https://huggingface.co/bigscience/bloom-560m/blob/main/tokenizer.json",
        "bigscience/bloom-1b1": "https://huggingface.co/bigscience/bloom-1b1/blob/main/tokenizer.json",
        "bigscience/bloom-1b7": "https://huggingface.co/bigscience/bloom-1b7/blob/main/tokenizer.json",
        "bigscience/bloom-3b": "https://huggingface.co/bigscience/bloom-3b/blob/main/tokenizer.json",
        "bigscience/bloom-7b1": "https://huggingface.co/bigscience/bloom-7b1/blob/main/tokenizer.json",
        "bigscience/bloom": "https://huggingface.co/bigscience/bloom/blob/main/tokenizer.json",
    },
}






class 			lowerCAmelCase__       (  lowerCamelCase_					):
					lowerCAmelCase_					       =					VOCAB_FILES_NAMES
					lowerCAmelCase_					       =					PRETRAINED_VOCAB_FILES_MAP
					lowerCAmelCase_					       =					['''input_ids''', '''attention_mask''']
					lowerCAmelCase_					       =					None






					def __init__(       self    ,							__SCREAMING_SNAKE_CASE=None    ,							__SCREAMING_SNAKE_CASE=None    ,							__SCREAMING_SNAKE_CASE=None    ,							__SCREAMING_SNAKE_CASE="<unk>"    ,							__SCREAMING_SNAKE_CASE="<s>"    ,							__SCREAMING_SNAKE_CASE="</s>"    ,							__SCREAMING_SNAKE_CASE="<pad>"    ,							__SCREAMING_SNAKE_CASE=False    ,							__SCREAMING_SNAKE_CASE=False    ,							**__SCREAMING_SNAKE_CASE    ,							):


											"""simple docstring"""





											super().__init__(
											    __SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							tokenizer_file=__SCREAMING_SNAKE_CASE    ,							unk_token=__SCREAMING_SNAKE_CASE    ,							bos_token=__SCREAMING_SNAKE_CASE    ,							eos_token=__SCREAMING_SNAKE_CASE    ,							pad_token=__SCREAMING_SNAKE_CASE    ,							add_prefix_space=__SCREAMING_SNAKE_CASE    ,							clean_up_tokenization_spaces=__SCREAMING_SNAKE_CASE    ,							**__SCREAMING_SNAKE_CASE    ,							)
											lowercase_     :					Dict													=	json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()		)
											if pre_tok_state.get('''add_prefix_space'''    ,							__SCREAMING_SNAKE_CASE		) != add_prefix_space:
																	lowercase_     :					Optional[int]													=	getattr(__SCREAMING_SNAKE_CASE    ,							pre_tok_state.pop('''type'''		)		)
																	lowercase_     :					List[Any]													=	add_prefix_space
																	lowercase_     :					Tuple													=	pre_tok_class(**__SCREAMING_SNAKE_CASE		)

											lowercase_     :					List[Any]													=	add_prefix_space






					def      _snake_case   (       self    ,							*__SCREAMING_SNAKE_CASE    ,							**__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_     :					Any													=	kwargs.get('''is_split_into_words'''    ,							__SCREAMING_SNAKE_CASE		)
											if not (self.add_prefix_space or not is_split_into_words):
																	raise Exception(
																	    F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with'''
																	    ''' pretokenized inputs.'''		)

											return super()._batch_encode_plus(*__SCREAMING_SNAKE_CASE    ,							**__SCREAMING_SNAKE_CASE		)






					def      _snake_case   (       self    ,							*__SCREAMING_SNAKE_CASE    ,							**__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_     :					Union[str, Any]													=	kwargs.get('''is_split_into_words'''    ,							__SCREAMING_SNAKE_CASE		)

											if not (self.add_prefix_space or not is_split_into_words):
																	raise Exception(
																	    F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True to use it with'''
																	    ''' pretokenized inputs.'''		)

											return super()._encode_plus(*__SCREAMING_SNAKE_CASE    ,							**__SCREAMING_SNAKE_CASE		)






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE = None		):


											"""simple docstring"""





											lowercase_     :					Union[str, Any]													=	self._tokenizer.model.save(__SCREAMING_SNAKE_CASE    ,							name=__SCREAMING_SNAKE_CASE		)
											return tuple(__SCREAMING_SNAKE_CASE		)






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_     :					Any													=	[]
											for is_user, text in conversation.iter_texts():
																	input_ids.extend(self.encode(__SCREAMING_SNAKE_CASE    ,							add_special_tokens=__SCREAMING_SNAKE_CASE		) + [self.eos_token_id]		)

											if len(__SCREAMING_SNAKE_CASE		) > self.model_max_length:
																	lowercase_     :					Any													=	input_ids[-self.model_max_length :]
											return input_ids





'''simple docstring'''







from ...configuration_utils import PretrainedConfig
from ...utils import logging


_lowercase			:  Any   							=	logging.get_logger(__name__)

_lowercase			:  Dict   							=	{
    "EleutherAI/gpt-neox-20b": "https://huggingface.co/EleutherAI/gpt-neox-20b/resolve/main/config.json",
    # See all GPTNeoX models at https://huggingface.co/models?filter=gpt_neox
}






class 			lowerCAmelCase__       (  lowerCamelCase_					):
					lowerCAmelCase_					       =					'''gpt_neox'''






					def __init__(       self    ,							__SCREAMING_SNAKE_CASE=5_04_32    ,							__SCREAMING_SNAKE_CASE=61_44    ,							__SCREAMING_SNAKE_CASE=44    ,							__SCREAMING_SNAKE_CASE=64    ,							__SCREAMING_SNAKE_CASE=2_45_76    ,							__SCREAMING_SNAKE_CASE="gelu"    ,							__SCREAMING_SNAKE_CASE=0.25    ,							__SCREAMING_SNAKE_CASE=1_00_00    ,							__SCREAMING_SNAKE_CASE=0.0    ,							__SCREAMING_SNAKE_CASE=0.0    ,							__SCREAMING_SNAKE_CASE=0.1    ,							__SCREAMING_SNAKE_CASE=20_48    ,							__SCREAMING_SNAKE_CASE=0.02    ,							__SCREAMING_SNAKE_CASE=1E-5    ,							__SCREAMING_SNAKE_CASE=True    ,							__SCREAMING_SNAKE_CASE=0    ,							__SCREAMING_SNAKE_CASE=2    ,							__SCREAMING_SNAKE_CASE=False    ,							__SCREAMING_SNAKE_CASE=True    ,							__SCREAMING_SNAKE_CASE=None    ,							**__SCREAMING_SNAKE_CASE    ,							):


											"""simple docstring"""





											super().__init__(bos_token_id=__SCREAMING_SNAKE_CASE    ,							eos_token_id=__SCREAMING_SNAKE_CASE    ,							**__SCREAMING_SNAKE_CASE		)
											lowercase_     :					Optional[Any]													=	vocab_size
											lowercase_     :					Optional[Any]													=	max_position_embeddings
											lowercase_     :					Optional[int]													=	hidden_size
											lowercase_     :					Tuple													=	num_hidden_layers
											lowercase_     :					Any													=	num_attention_heads
											lowercase_     :					str													=	intermediate_size
											lowercase_     :					Any													=	hidden_act
											lowercase_     :					Tuple													=	rotary_pct
											lowercase_     :					Optional[Any]													=	rotary_emb_base
											lowercase_     :					Any													=	attention_dropout
											lowercase_     :					str													=	hidden_dropout
											lowercase_     :					Dict													=	classifier_dropout
											lowercase_     :					Tuple													=	initializer_range
											lowercase_     :					List[str]													=	layer_norm_eps
											lowercase_     :					Union[str, Any]													=	use_cache
											lowercase_     :					int													=	tie_word_embeddings
											lowercase_     :					Tuple													=	use_parallel_residual
											lowercase_     :					Optional[Any]													=	rope_scaling
											self._rope_scaling_validation()

											if self.hidden_size % self.num_attention_heads != 0:
																	raise ValueError(
																	    '''The hidden size is not divisble by the number of attention heads! Make sure to update them!'''		)






					def      _snake_case   (       self		):


											"""simple docstring"""





											if self.rope_scaling is None:
																	return

											if not isinstance(self.rope_scaling    ,							__SCREAMING_SNAKE_CASE		) or len(self.rope_scaling		) != 2:
																	raise ValueError(
																	    '''`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '''
																	    F'''got {self.rope_scaling}'''		)
											lowercase_     :					List[Any]													=	self.rope_scaling.get('''type'''    ,							__SCREAMING_SNAKE_CASE		)
											lowercase_     :					int													=	self.rope_scaling.get('''factor'''    ,							__SCREAMING_SNAKE_CASE		)
											if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
																	raise ValueError(
																	    F'''`rope_scaling`\'s name field must be one of [\'linear\', \'dynamic\'], got {rope_scaling_type}'''		)
											if rope_scaling_factor is None or not isinstance(__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE		) or rope_scaling_factor <= 1.0:
																	raise ValueError(F'''`rope_scaling`\'s factor field must be an float > 1, got {rope_scaling_factor}'''		)



