






import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_gpta import GPTaTokenizer


if TYPE_CHECKING:
	from transformers.pipelines.conversational import Conversation


_UpperCamelCase			:	Union[str, Any]								=logging.get_logger(__name__)

_UpperCamelCase			:	Union[str, Any]								={'vocab_file': 'vocab.json', 'merges_file': 'merges.txt', 'tokenizer_file': 'tokenizer.json'}

_UpperCamelCase			:	Optional[Any]								={
    'vocab_file': {
        'gpt2': 'https://huggingface.co/gpt2/resolve/main/vocab.json',
        'gpt2-medium': 'https://huggingface.co/gpt2-medium/resolve/main/vocab.json',
        'gpt2-large': 'https://huggingface.co/gpt2-large/resolve/main/vocab.json',
        'gpt2-xl': 'https://huggingface.co/gpt2-xl/resolve/main/vocab.json',
        'distilgpt2': 'https://huggingface.co/distilgpt2/resolve/main/vocab.json',
    },
    'merges_file': {
        'gpt2': 'https://huggingface.co/gpt2/resolve/main/merges.txt',
        'gpt2-medium': 'https://huggingface.co/gpt2-medium/resolve/main/merges.txt',
        'gpt2-large': 'https://huggingface.co/gpt2-large/resolve/main/merges.txt',
        'gpt2-xl': 'https://huggingface.co/gpt2-xl/resolve/main/merges.txt',
        'distilgpt2': 'https://huggingface.co/distilgpt2/resolve/main/merges.txt',
    },
    'tokenizer_file': {
        'gpt2': 'https://huggingface.co/gpt2/resolve/main/tokenizer.json',
        'gpt2-medium': 'https://huggingface.co/gpt2-medium/resolve/main/tokenizer.json',
        'gpt2-large': 'https://huggingface.co/gpt2-large/resolve/main/tokenizer.json',
        'gpt2-xl': 'https://huggingface.co/gpt2-xl/resolve/main/tokenizer.json',
        'distilgpt2': 'https://huggingface.co/distilgpt2/resolve/main/tokenizer.json',
    },
}

_UpperCamelCase			:	Optional[int]								={
    'gpt2': 1024,
    'gpt2-medium': 1024,
    'gpt2-large': 1024,
    'gpt2-xl': 1024,
    'distilgpt2': 1024,
}






class 						UpperCAmelCase__ (							__snake_case							):
		__snake_case      :       Union[str, Any]     =   VOCAB_FILES_NAMES
		__snake_case      :       str     =   PRETRAINED_VOCAB_FILES_MAP
		__snake_case      :       Union[str, Any]     =   PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
		__snake_case      :       Optional[Any]     =   ["input_ids", "attention_mask"]
		__snake_case      :       Optional[Any]     =   GPTaTokenizer







		def __init__(					self       ,A__=None       ,A__=None       ,A__=None       ,A__="<|endoftext|>"       ,A__="<|endoftext|>"       ,A__="<|endoftext|>"       ,A__=False       ,**A__       ,):
						super().__init__(
						    A__       ,A__       ,tokenizer_file=A__       ,unk_token=A__       ,bos_token=A__       ,eos_token=A__       ,add_prefix_space=A__       ,**A__       ,)

						_A	:				Tuple   =      kwargs.pop('''add_bos_token'''       ,A__		)

						_A	:				List[str]   =      json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()		)
						if pre_tok_state.get('''add_prefix_space'''       ,A__		) != add_prefix_space:
										_A	:				Union[str, Any]   =      getattr(A__       ,pre_tok_state.pop('''type'''		)		)
										_A	:				Optional[Any]   =      add_prefix_space
										_A	:				Tuple   =      pre_tok_class(**A__		)

						_A	:				Optional[int]   =      add_prefix_space







		def  A__      (					self       ,*A__       ,**A__		):
						_A	:				Dict   =      kwargs.get('''is_split_into_words'''       ,A__		)
						assert self.add_prefix_space or not is_split_into_words, (
						    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
						    "to use it with pretokenized inputs."
						)

						return super()._batch_encode_plus(*A__       ,**A__		)







		def  A__      (					self       ,*A__       ,**A__		):
						_A	:				int   =      kwargs.get('''is_split_into_words'''       ,A__		)

						assert self.add_prefix_space or not is_split_into_words, (
						    f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
						    "to use it with pretokenized inputs."
						)

						return super()._encode_plus(*A__       ,**A__		)







		def  A__      (					self       ,A__       ,A__ = None		):
						_A	:				Optional[int]   =      self._tokenizer.model.save(A__       ,name=A__		)
						return tuple(A__		)







		def  A__      (					self       ,A__		):
						_A	:				Dict   =      []
						for is_user, text in conversation.iter_texts():
										input_ids.extend(self.encode(A__       ,add_special_tokens=A__		) + [self.eos_token_id]		)

						if len(A__		) > self.model_max_length:
										_A	:				Dict   =      input_ids[-self.model_max_length :]
						return input_ids











from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


_UpperCamelCase			:	Any								={
    'configuration_megatron_bert': ['MEGATRON_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP', 'MegatronBertConfig'],
}

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	_UpperCamelCase			:	Optional[Any]								=[
	    'MEGATRON_BERT_PRETRAINED_MODEL_ARCHIVE_LIST',
	    'MegatronBertForCausalLM',
	    'MegatronBertForMaskedLM',
	    'MegatronBertForMultipleChoice',
	    'MegatronBertForNextSentencePrediction',
	    'MegatronBertForPreTraining',
	    'MegatronBertForQuestionAnswering',
	    'MegatronBertForSequenceClassification',
	    'MegatronBertForTokenClassification',
	    'MegatronBertModel',
	    'MegatronBertPreTrainedModel',
	]

if TYPE_CHECKING:
	from .configuration_megatron_bert import MEGATRON_BERT_PRETRAINED_CONFIG_ARCHIVE_MAP, MegatronBertConfig

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_megatron_bert import (
		    MEGATRON_BERT_PRETRAINED_MODEL_ARCHIVE_LIST,
		    MegatronBertForCausalLM,
		    MegatronBertForMaskedLM,
		    MegatronBertForMultipleChoice,
		    MegatronBertForNextSentencePrediction,
		    MegatronBertForPreTraining,
		    MegatronBertForQuestionAnswering,
		    MegatronBertForSequenceClassification,
		    MegatronBertForTokenClassification,
		    MegatronBertModel,
		    MegatronBertPreTrainedModel,
		)

else:
	import sys

	_UpperCamelCase			:	Optional[int]								=_LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)



