
'''simple docstring'''







import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_gpta import GPTaTokenizer


if TYPE_CHECKING:
							from transformers.pipelines.conversational import Conversation


_lowercase			:  Any   							=	logging.get_logger(__name__)

_lowercase			:  List[Any]   							=	{"vocab_file": "vocab.json", "merges_file": "merges.txt", "tokenizer_file": "tokenizer.json"}

_lowercase			:  Optional[int]   							=	{
    "vocab_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/vocab.json",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/vocab.json",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/vocab.json",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/vocab.json",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/vocab.json",
    },
    "merges_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/merges.txt",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/merges.txt",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/merges.txt",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/merges.txt",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/merges.txt",
    },
    "tokenizer_file": {
        "gpt2": "https://huggingface.co/gpt2/resolve/main/tokenizer.json",
        "gpt2-medium": "https://huggingface.co/gpt2-medium/resolve/main/tokenizer.json",
        "gpt2-large": "https://huggingface.co/gpt2-large/resolve/main/tokenizer.json",
        "gpt2-xl": "https://huggingface.co/gpt2-xl/resolve/main/tokenizer.json",
        "distilgpt2": "https://huggingface.co/distilgpt2/resolve/main/tokenizer.json",
    },
}

_lowercase			:  List[Any]   							=	{
    "gpt2": 1_0_2_4,
    "gpt2-medium": 1_0_2_4,
    "gpt2-large": 1_0_2_4,
    "gpt2-xl": 1_0_2_4,
    "distilgpt2": 1_0_2_4,
}






class 			lowerCAmelCase__       (  lowerCamelCase_					):
					lowerCAmelCase_					       =					VOCAB_FILES_NAMES
					lowerCAmelCase_					       =					PRETRAINED_VOCAB_FILES_MAP
					lowerCAmelCase_					       =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowerCAmelCase_					       =					['''input_ids''', '''attention_mask''']
					lowerCAmelCase_					       =					GPTaTokenizer






					def __init__(       self    ,							__SCREAMING_SNAKE_CASE=None    ,							__SCREAMING_SNAKE_CASE=None    ,							__SCREAMING_SNAKE_CASE=None    ,							__SCREAMING_SNAKE_CASE="<|endoftext|>"    ,							__SCREAMING_SNAKE_CASE="<|endoftext|>"    ,							__SCREAMING_SNAKE_CASE="<|endoftext|>"    ,							__SCREAMING_SNAKE_CASE=False    ,							**__SCREAMING_SNAKE_CASE    ,							):


											"""simple docstring"""





											super().__init__(
											    __SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							tokenizer_file=__SCREAMING_SNAKE_CASE    ,							unk_token=__SCREAMING_SNAKE_CASE    ,							bos_token=__SCREAMING_SNAKE_CASE    ,							eos_token=__SCREAMING_SNAKE_CASE    ,							add_prefix_space=__SCREAMING_SNAKE_CASE    ,							**__SCREAMING_SNAKE_CASE    ,							)

											lowercase_     :					Dict													=	kwargs.pop('''add_bos_token'''    ,							__SCREAMING_SNAKE_CASE		)

											lowercase_     :					str													=	json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()		)
											if pre_tok_state.get('''add_prefix_space'''    ,							__SCREAMING_SNAKE_CASE		) != add_prefix_space:
																	lowercase_     :					Dict													=	getattr(__SCREAMING_SNAKE_CASE    ,							pre_tok_state.pop('''type'''		)		)
																	lowercase_     :					str													=	add_prefix_space
																	lowercase_     :					Optional[int]													=	pre_tok_class(**__SCREAMING_SNAKE_CASE		)

											lowercase_     :					Union[str, Any]													=	add_prefix_space






					def      _snake_case   (       self    ,							*__SCREAMING_SNAKE_CASE    ,							**__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_     :					str													=	kwargs.get('''is_split_into_words'''    ,							__SCREAMING_SNAKE_CASE		)
											assert self.add_prefix_space or not is_split_into_words, (
											    F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
											    "to use it with pretokenized inputs."
											)

											return super()._batch_encode_plus(*__SCREAMING_SNAKE_CASE    ,							**__SCREAMING_SNAKE_CASE		)






					def      _snake_case   (       self    ,							*__SCREAMING_SNAKE_CASE    ,							**__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_     :					List[Any]													=	kwargs.get('''is_split_into_words'''    ,							__SCREAMING_SNAKE_CASE		)

											assert self.add_prefix_space or not is_split_into_words, (
											    F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
											    "to use it with pretokenized inputs."
											)

											return super()._encode_plus(*__SCREAMING_SNAKE_CASE    ,							**__SCREAMING_SNAKE_CASE		)






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE = None		):


											"""simple docstring"""





											lowercase_     :					Tuple													=	self._tokenizer.model.save(__SCREAMING_SNAKE_CASE    ,							name=__SCREAMING_SNAKE_CASE		)
											return tuple(__SCREAMING_SNAKE_CASE		)






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_     :					Optional[Any]													=	[]
											for is_user, text in conversation.iter_texts():
																	input_ids.extend(self.encode(__SCREAMING_SNAKE_CASE    ,							add_special_tokens=__SCREAMING_SNAKE_CASE		) + [self.eos_token_id]		)

											if len(__SCREAMING_SNAKE_CASE		) > self.model_max_length:
																	lowercase_     :					Optional[int]													=	input_ids[-self.model_max_length :]
											return input_ids





'''simple docstring'''







import logging
import os

from .state import PartialState






class 			lowerCAmelCase__       (  logging.LoggerAdapter					):






					@staticmethod
					def      _snake_case   (       __SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											lowercase_     :					Optional[Any]													=	PartialState()
											return not main_process_only or (main_process_only and state.is_main_process)






					def      _snake_case   (       self    ,							__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							*__SCREAMING_SNAKE_CASE    ,							**__SCREAMING_SNAKE_CASE		):


											"""simple docstring"""





											if PartialState._shared_state == {}:
																	raise RuntimeError(
																	    '''You must initialize the accelerate state by calling either `PartialState()` or `Accelerator()` before using the logging utility.'''		)
											lowercase_     :					Tuple													=	kwargs.pop('''main_process_only'''    ,							__SCREAMING_SNAKE_CASE		)
											lowercase_     :					Optional[int]													=	kwargs.pop('''in_order'''    ,							__SCREAMING_SNAKE_CASE		)

											if self.isEnabledFor(__SCREAMING_SNAKE_CASE		):
																	if self._should_log(__SCREAMING_SNAKE_CASE		):
																							lowercase_						,			lowercase_     :					Optional[Any]													=	self.process(__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE		)
																							self.logger.log(__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							*__SCREAMING_SNAKE_CASE    ,							**__SCREAMING_SNAKE_CASE		)

																	elif in_order:
																							lowercase_     :					Optional[Any]													=	PartialState()
																							for i in range(state.num_processes		):
																													if i == state.process_index:
																																			lowercase_						,			lowercase_     :					Optional[int]													=	self.process(__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE		)
																																			self.logger.log(__SCREAMING_SNAKE_CASE    ,							__SCREAMING_SNAKE_CASE    ,							*__SCREAMING_SNAKE_CASE    ,							**__SCREAMING_SNAKE_CASE		)
																													state.wait_for_everyone()







def  snake_case_		(    __SCREAMING_SNAKE_CASE :			str				,				__SCREAMING_SNAKE_CASE :			str = None							):



						"""simple docstring"""





						if log_level is None:
												lowercase_     :					Any													=	os.environ.get('''ACCELERATE_LOG_LEVEL'''				,				__SCREAMING_SNAKE_CASE							)
						lowercase_     :					List[str]													=	logging.getLogger(__SCREAMING_SNAKE_CASE							)
						if log_level is not None:
												logger.setLevel(log_level.upper()							)
												logger.root.setLevel(log_level.upper()							)
						return MultiProcessAdapter(__SCREAMING_SNAKE_CASE				,				{}							)



