"""simple docstring"""



import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_gpta import GPTaTokenizer


if TYPE_CHECKING:
				from transformers.pipelines.conversational import Conversation


_UpperCamelCase								=		logging.get_logger(__name__)

_UpperCamelCase								=		{"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

_UpperCamelCase								=		{
    """vocab_file""": {
        """gpt2""": """https://huggingface.co/gpt2/resolve/main/vocab.json""",
        """gpt2-medium""": """https://huggingface.co/gpt2-medium/resolve/main/vocab.json""",
        """gpt2-large""": """https://huggingface.co/gpt2-large/resolve/main/vocab.json""",
        """gpt2-xl""": """https://huggingface.co/gpt2-xl/resolve/main/vocab.json""",
        """distilgpt2""": """https://huggingface.co/distilgpt2/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """gpt2""": """https://huggingface.co/gpt2/resolve/main/merges.txt""",
        """gpt2-medium""": """https://huggingface.co/gpt2-medium/resolve/main/merges.txt""",
        """gpt2-large""": """https://huggingface.co/gpt2-large/resolve/main/merges.txt""",
        """gpt2-xl""": """https://huggingface.co/gpt2-xl/resolve/main/merges.txt""",
        """distilgpt2""": """https://huggingface.co/distilgpt2/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """gpt2""": """https://huggingface.co/gpt2/resolve/main/tokenizer.json""",
        """gpt2-medium""": """https://huggingface.co/gpt2-medium/resolve/main/tokenizer.json""",
        """gpt2-large""": """https://huggingface.co/gpt2-large/resolve/main/tokenizer.json""",
        """gpt2-xl""": """https://huggingface.co/gpt2-xl/resolve/main/tokenizer.json""",
        """distilgpt2""": """https://huggingface.co/distilgpt2/resolve/main/tokenizer.json""",
    },
}

_UpperCamelCase								=		{
    """gpt2""": 10_24,
    """gpt2-medium""": 10_24,
    """gpt2-large""": 10_24,
    """gpt2-xl""": 10_24,
    """distilgpt2""": 10_24,
}
class 						__UpperCAmelCase    (SCREAMING_SNAKE_CASE_     ):







			'''simple docstring'''
			_UpperCamelCase  :						Optional[int]					     =				VOCAB_FILES_NAMES
			_UpperCamelCase  :						Optional[int]					     =				PRETRAINED_VOCAB_FILES_MAP
			_UpperCamelCase  :						str					     =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			_UpperCamelCase  :						Tuple					     =				['input_ids', 'attention_mask']
			_UpperCamelCase  :						Union[str, Any]					     =				GPTaTokenizer





			def __init__(			self   ,      snake_case_=None   ,      snake_case_=None   ,      snake_case_=None   ,      snake_case_="<|endoftext|>"   ,      snake_case_="<|endoftext|>"   ,      snake_case_="<|endoftext|>"   ,      snake_case_=False   ,      **snake_case_   ,      ):







										'''simple docstring'''


										super().__init__(
										    snake_case__   ,      snake_case__   ,      tokenizer_file=snake_case__   ,      unk_token=snake_case__   ,      bos_token=snake_case__   ,      eos_token=snake_case__   ,      add_prefix_space=snake_case__   ,      **snake_case__   ,      )

										A__       :     str								=			kwargs.pop("""add_bos_token"""   ,      snake_case__							)

										A__       :     List[Any]								=			json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()							)
										if pre_tok_state.get("""add_prefix_space"""   ,      snake_case__							) != add_prefix_space:
																	A__       :     str								=			getattr(snake_case__   ,      pre_tok_state.pop("""type"""							)							)
																	A__       :     List[str]								=			add_prefix_space
																	A__       :     Dict								=			pre_tok_class(**snake_case__							)

										A__       :     Any								=			add_prefix_space





			def 		lowerCamelCase					(			self   ,      *snake_case_   ,      **snake_case_							):







										'''simple docstring'''


										A__       :     int								=			kwargs.get("""is_split_into_words"""   ,      snake_case__							)
										assert self.add_prefix_space or not is_split_into_words, (
										    F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
										    "to use it with pretokenized inputs."
										)

										return super()._batch_encode_plus(*snake_case__   ,      **snake_case__							)





			def 		lowerCamelCase					(			self   ,      *snake_case_   ,      **snake_case_							):







										'''simple docstring'''


										A__       :     int								=			kwargs.get("""is_split_into_words"""   ,      snake_case__							)

										assert self.add_prefix_space or not is_split_into_words, (
										    F'''You need to instantiate {self.__class__.__name__} with add_prefix_space=True '''
										    "to use it with pretokenized inputs."
										)

										return super()._encode_plus(*snake_case__   ,      **snake_case__							)





			def 		lowerCamelCase					(			self   ,      snake_case_   ,      snake_case_ = None							):







										'''simple docstring'''


										A__       :     Optional[int]								=			self._tokenizer.model.save(snake_case__   ,      name=snake_case__							)
										return tuple(snake_case__							)





			def 		lowerCamelCase					(			self   ,      snake_case_							):







										'''simple docstring'''


										A__       :     Any								=			[]
										for is_user, text in conversation.iter_texts():
																	input_ids.extend(self.encode(snake_case__   ,      add_special_tokens=snake_case__							) + [self.eos_token_id]							)

										if len(snake_case__							) > self.model_max_length:
																	A__       :     str								=			input_ids[-self.model_max_length :]
										return input_ids









'''simple docstring'''







from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


lowerCAmelCase     :  List[str]											=    {
    """configuration_electra""": ["""ELECTRA_PRETRAINED_CONFIG_ARCHIVE_MAP""", """ElectraConfig""", """ElectraOnnxConfig"""],
    """tokenization_electra""": ["""ElectraTokenizer"""],
}

try:
       if not is_tokenizers_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       lowerCAmelCase     :  Optional[Any]											=    ["""ElectraTokenizerFast"""]

try:
       if not is_torch_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       lowerCAmelCase     :  str											=    [
           """ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST""",
           """ElectraForCausalLM""",
           """ElectraForMaskedLM""",
           """ElectraForMultipleChoice""",
           """ElectraForPreTraining""",
           """ElectraForQuestionAnswering""",
           """ElectraForSequenceClassification""",
           """ElectraForTokenClassification""",
           """ElectraModel""",
           """ElectraPreTrainedModel""",
           """load_tf_weights_in_electra""",
       ]

try:
       if not is_tf_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       lowerCAmelCase     :  Tuple											=    [
           """TF_ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST""",
           """TFElectraForMaskedLM""",
           """TFElectraForMultipleChoice""",
           """TFElectraForPreTraining""",
           """TFElectraForQuestionAnswering""",
           """TFElectraForSequenceClassification""",
           """TFElectraForTokenClassification""",
           """TFElectraModel""",
           """TFElectraPreTrainedModel""",
       ]

try:
       if not is_flax_available():
              raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
       pass
else:
       lowerCAmelCase     :  Union[str, Any]											=    [
           """FlaxElectraForCausalLM""",
           """FlaxElectraForMaskedLM""",
           """FlaxElectraForMultipleChoice""",
           """FlaxElectraForPreTraining""",
           """FlaxElectraForQuestionAnswering""",
           """FlaxElectraForSequenceClassification""",
           """FlaxElectraForTokenClassification""",
           """FlaxElectraModel""",
           """FlaxElectraPreTrainedModel""",
       ]


if TYPE_CHECKING:
       from .configuration_electra import ELECTRA_PRETRAINED_CONFIG_ARCHIVE_MAP, ElectraConfig, ElectraOnnxConfig
       from .tokenization_electra import ElectraTokenizer

       try:
              if not is_tokenizers_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .tokenization_electra_fast import ElectraTokenizerFast

       try:
              if not is_torch_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_electra import (
                  ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST,
                  ElectraForCausalLM,
                  ElectraForMaskedLM,
                  ElectraForMultipleChoice,
                  ElectraForPreTraining,
                  ElectraForQuestionAnswering,
                  ElectraForSequenceClassification,
                  ElectraForTokenClassification,
                  ElectraModel,
                  ElectraPreTrainedModel,
                  load_tf_weights_in_electra,
              )

       try:
              if not is_tf_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_tf_electra import (
                  TF_ELECTRA_PRETRAINED_MODEL_ARCHIVE_LIST,
                  TFElectraForMaskedLM,
                  TFElectraForMultipleChoice,
                  TFElectraForPreTraining,
                  TFElectraForQuestionAnswering,
                  TFElectraForSequenceClassification,
                  TFElectraForTokenClassification,
                  TFElectraModel,
                  TFElectraPreTrainedModel,
              )

       try:
              if not is_flax_available():
                     raise OptionalDependencyNotAvailable()
       except OptionalDependencyNotAvailable:
              pass
       else:
              from .modeling_flax_electra import (
                  FlaxElectraForCausalLM,
                  FlaxElectraForMaskedLM,
                  FlaxElectraForMultipleChoice,
                  FlaxElectraForPreTraining,
                  FlaxElectraForQuestionAnswering,
                  FlaxElectraForSequenceClassification,
                  FlaxElectraForTokenClassification,
                  FlaxElectraModel,
                  FlaxElectraPreTrainedModel,
              )

else:
       import sys

       lowerCAmelCase     :  int											=    _LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)

