
'''simple docstring'''







import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_gpta import GPTaTokenizer


if TYPE_CHECKING:
							from transformers.pipelines.conversational import Conversation


_lowercase			:  Any   							=	logging.get_logger(__name__)

_lowercase			:  List[Any]   							=	{"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

_lowercase			:  Optional[int]   							=	{
    "vocab_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/vocab.json",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/vocab.json",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/vocab.json",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/vocab.json",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/vocab.json",
    },
    "merges_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/merges.txt",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/merges.txt",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/merges.txt",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/merges.txt",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/merges.txt",
    },
    "tokenizer_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/tokenizer.json",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/tokenizer.json",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/tokenizer.json",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/tokenizer.json",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/tokenizer.json",
    },
}

_lowercase			:  List[Any]   							=	{
    "gpt2": 1_0_2_4,
    "gpt2-medium": 1_0_2_4,
    "gpt2-large": 1_0_2_4,
    "gpt2-xl": 1_0_2_4,
    "distilgpt2": 1_0_2_4,
}






class 			SCREAMING_SNAKE_CASE__       (  lowerCamelCase_					):
					lowerCAmelCase_					       =					VOCAB_FILES_NAMES
					lowerCAmelCase_					       =					PRETRAINED_VOCAB_FILES_MAP
					lowerCAmelCase_					       =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowerCAmelCase_					       =					['''input_ids''', '''attention_mask''']
					lowerCAmelCase_					       =					GPTaTokenizer






					def __init__(       self    ,							__SCREAMING_SNAKE_CASE=None    ,							__SCREAMING_SNAKE_CASE=None    ,							__SCREAMING_SNAKE_CASE=None    ,							__SCREAMING_SNAKE_CASE="<|endoftext|>"    ,							__SCREAMING_SNAKE_CASE="<|endoftext|>"    ,							__SCREAMING_SNAKE_CASE="<|endoftext|>"    ,							__SCREAMING_SNAKE_CASE=False    ,							**__SCREAMING_SNAKE_CASE    ,							):


											"""simple docstring"""





											super().__init__(
											    __SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							tokenizer_file=__SCREAMING_SNAKE_CASE    ,							unk_token=__SCREAMING_SNAKE_CASE    ,							bos_token=__SCREAMING_SNAKE_CASE    ,							eos_token=__SCREAMING_SNAKE_CASE    ,							add_prefix_space=__SCREAMING_SNAKE_CASE    ,							**__SCREAMING_SNAKE_CASE    ,							)

											lowercase_     :					Dict													=	kwargs.pop('''add_bos_token'''    ,							__SCREAMING_SNAKE_CASE		)

											lowercase_     :					str													=	json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()		)
											if pre_tok_state.get('''add_prefix_space'''    ,							__SCREAMING_SNAKE_CASE		) != add_prefix_space:
																	lowercase_     :					Dict													=	getattr(__SCREAMING_SNAKE_CASE    ,							pre_tok_state.pop('''type'''		)		)
																	lowercase_     :					str													=	add_prefix_space
																	lowercase_     :					Optional[int]													=	pre_tok_class(**__SCREAMING_SNAKE_CASE		)

											lowercase_     :					Union[str, Any]													=	add_prefix_space






					def      _snake_case   (       self    ,							*__SCREAMING_SNAKE_CASE    ,							**__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_     :					str													=	kwargs.get('''is_split_into_words'''    ,							__SCREAMING_SNAKE_CASE		)
											assert self.add_prefix_space or not is_split_into_words, (
											    F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
											    "to use it with pretokenized inputs."
											)

											return super()._batch_encode_plus(*__SCREAMING_SNAKE_CASE    ,							**__SCREAMING_SNAKE_CASE		)






					def      _snake_case   (       self    ,							*__SCREAMING_SNAKE_CASE    ,							**__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_     :					List[Any]													=	kwargs.get('''is_split_into_words'''    ,							__SCREAMING_SNAKE_CASE		)

											assert self.add_prefix_space or not is_split_into_words, (
											    F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
											    "to use it with pretokenized inputs."
											)

											return super()._encode_plus(*__SCREAMING_SNAKE_CASE    ,							**__SCREAMING_SNAKE_CASE		)






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE = None		):


											"""simple docstring"""





											lowercase_     :					Tuple													=	self._tokenizer.model.save(__SCREAMING_SNAKE_CASE    ,							name=__SCREAMING_SNAKE_CASE		)
											return tuple(__SCREAMING_SNAKE_CASE		)






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_     :					Optional[Any]													=	[]
											for is_user, text in conversation.iter_texts():
																	input_ids.extend(self.encode(__SCREAMING_SNAKE_CASE    ,							add_special_tokens=__SCREAMING_SNAKE_CASE		) + [self.eos_token_id]		)

											if len(__SCREAMING_SNAKE_CASE		) > self.model_max_length:
																	lowercase_     :					Optional[int]													=	input_ids[-self.model_max_length :]
											return input_ids





'''simple docstring'''







def  snake_case_		(    __SCREAMING_SNAKE_CASE :			str							):



						"""simple docstring"""





						assert column_title.isupper()
						lowercase_     :					Dict													=	0
						lowercase_     :					Tuple													=	len(__SCREAMING_SNAKE_CASE							) - 1
						lowercase_     :					Optional[int]													=	0

						while index >= 0:
												lowercase_     :					Optional[Any]													=	(ord(column_title[index]							) - 64) * pow(26				,				__SCREAMING_SNAKE_CASE							)
												answer += value
												power += 1
												index -= 1

						return answer


if __name__ == "__main__":
							from doctest import testmod

							testmod()



