






'''simple docstring'''



import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_gpta import GPTaTokenizer


if TYPE_CHECKING:
 from transformers.pipelines.conversational import Conversation


__UpperCamelCase      :       Tuple      			=      logging.get_logger(__name__)

__UpperCamelCase      :       Dict      			=      {"""vocab_file""": """vocab.json""", """merges_file""": """merges.txt""", """tokenizer_file""": """tokenizer.json"""}

__UpperCamelCase      :       Any      			=      {
    """vocab_file""": {
        """gpt2""": """https://huggingface.co/gpt2/resolve/main/vocab.json""",
        """gpt2-medium""": """https://huggingface.co/gpt2-medium/resolve/main/vocab.json""",
        """gpt2-large""": """https://huggingface.co/gpt2-large/resolve/main/vocab.json""",
        """gpt2-xl""": """https://huggingface.co/gpt2-xl/resolve/main/vocab.json""",
        """distilgpt2""": """https://huggingface.co/distilgpt2/resolve/main/vocab.json""",
    },
    """merges_file""": {
        """gpt2""": """https://huggingface.co/gpt2/resolve/main/merges.txt""",
        """gpt2-medium""": """https://huggingface.co/gpt2-medium/resolve/main/merges.txt""",
        """gpt2-large""": """https://huggingface.co/gpt2-large/resolve/main/merges.txt""",
        """gpt2-xl""": """https://huggingface.co/gpt2-xl/resolve/main/merges.txt""",
        """distilgpt2""": """https://huggingface.co/distilgpt2/resolve/main/merges.txt""",
    },
    """tokenizer_file""": {
        """gpt2""": """https://huggingface.co/gpt2/resolve/main/tokenizer.json""",
        """gpt2-medium""": """https://huggingface.co/gpt2-medium/resolve/main/tokenizer.json""",
        """gpt2-large""": """https://huggingface.co/gpt2-large/resolve/main/tokenizer.json""",
        """gpt2-xl""": """https://huggingface.co/gpt2-xl/resolve/main/tokenizer.json""",
        """distilgpt2""": """https://huggingface.co/distilgpt2/resolve/main/tokenizer.json""",
    },
}

__UpperCamelCase      :       Tuple      			=      {
    """gpt2""": 1024,
    """gpt2-medium""": 1024,
    """gpt2-large""": 1024,
    """gpt2-xl""": 1024,
    """distilgpt2""": 1024,
}
class 							__SCREAMING_SNAKE_CASE       (							_lowerCAmelCase     ):
      __a  					=VOCAB_FILES_NAMES
      __a  					=PRETRAINED_VOCAB_FILES_MAP
      __a  					=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      __a  					=["input_ids", "attention_mask"]
      __a  					=GPTaTokenizer




      def __init__(  self ,   lowerCamelCase=None ,   lowerCamelCase=None ,   lowerCamelCase=None ,   lowerCamelCase="<|endoftext|>" ,   lowerCamelCase="<|endoftext|>" ,   lowerCamelCase="<|endoftext|>" ,   lowerCamelCase=False ,   **lowerCamelCase ,   )							->List[str]:



         '''simple docstring'''
         super().__init__(
             lowerCamelCase ,   lowerCamelCase ,   tokenizer_file=lowerCamelCase ,   unk_token=lowerCamelCase ,   bos_token=lowerCamelCase ,   eos_token=lowerCamelCase ,   add_prefix_space=lowerCamelCase ,   **lowerCamelCase ,   )

         __a				     =    kwargs.pop('add_bos_token' ,   lowerCamelCase      )

         __a				     =    json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()      )
         if pre_tok_state.get('add_prefix_space' ,   lowerCamelCase      ) != add_prefix_space:
            __a				     =    getattr(lowerCamelCase ,   pre_tok_state.pop('type'      )      )
            __a				     =    add_prefix_space
            __a				     =    pre_tok_class(**lowerCamelCase      )

         __a				     =    add_prefix_space




      def  __UpperCamelCase							(  self ,   *lowerCamelCase ,   **lowerCamelCase      )							->BatchEncoding:



         '''simple docstring'''
         __a				     =    kwargs.get('is_split_into_words' ,   lowerCamelCase      )
         assert self.add_prefix_space or not is_split_into_words, (
             F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
             "to use it with pretokenized inputs."
         )

         return super()._batch_encode_plus(*lowerCamelCase ,   **lowerCamelCase      )




      def  __UpperCamelCase							(  self ,   *lowerCamelCase ,   **lowerCamelCase      )							->BatchEncoding:



         '''simple docstring'''
         __a				     =    kwargs.get('is_split_into_words' ,   lowerCamelCase      )

         assert self.add_prefix_space or not is_split_into_words, (
             F"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
             "to use it with pretokenized inputs."
         )

         return super()._encode_plus(*lowerCamelCase ,   **lowerCamelCase      )




      def  __UpperCamelCase							(  self ,   lowerCamelCase ,   lowerCamelCase = None      )							->Tuple[str]:



         '''simple docstring'''
         __a				     =    self._tokenizer.model.save(lowerCamelCase ,   name=lowerCamelCase      )
         return tuple(lowerCamelCase      )






      def  __UpperCamelCase							(  self ,   lowerCamelCase      )							->List[int]:



         '''simple docstring'''
         __a				     =    []
         for is_user, text in conversation.iter_texts():
            input_ids.extend(self.encode(lowerCamelCase ,   add_special_tokens=lowerCamelCase      ) + [self.eos_token_id]      )

         if len(lowerCamelCase      ) > self.model_max_length:
            __a				     =    input_ids[-self.model_max_length :]
         return input_ids







'''simple docstring'''



from collections import OrderedDict
from typing import Mapping

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


__UpperCamelCase      :       Dict      			=      logging.get_logger(__name__)

__UpperCamelCase      :       Optional[int]      			=      {
    """facebook/xmod-base""": """https://huggingface.co/facebook/xmod-base/resolve/main/config.json""",
    """facebook/xmod-large-prenorm""": """https://huggingface.co/facebook/xmod-large-prenorm/resolve/main/config.json""",
    """facebook/xmod-base-13-125k""": """https://huggingface.co/facebook/xmod-base-13-125k/resolve/main/config.json""",
    """facebook/xmod-base-30-125k""": """https://huggingface.co/facebook/xmod-base-30-125k/resolve/main/config.json""",
    """facebook/xmod-base-30-195k""": """https://huggingface.co/facebook/xmod-base-30-195k/resolve/main/config.json""",
    """facebook/xmod-base-60-125k""": """https://huggingface.co/facebook/xmod-base-60-125k/resolve/main/config.json""",
    """facebook/xmod-base-60-265k""": """https://huggingface.co/facebook/xmod-base-60-265k/resolve/main/config.json""",
    """facebook/xmod-base-75-125k""": """https://huggingface.co/facebook/xmod-base-75-125k/resolve/main/config.json""",
    """facebook/xmod-base-75-269k""": """https://huggingface.co/facebook/xmod-base-75-269k/resolve/main/config.json""",
}
class 							__SCREAMING_SNAKE_CASE       (							_lowerCAmelCase     ):
      __a  					="xmod"




      def __init__(  self ,   lowerCamelCase=3_0522 ,   lowerCamelCase=768 ,   lowerCamelCase=12 ,   lowerCamelCase=12 ,   lowerCamelCase=3072 ,   lowerCamelCase="gelu" ,   lowerCamelCase=0.1 ,   lowerCamelCase=0.1 ,   lowerCamelCase=512 ,   lowerCamelCase=2 ,   lowerCamelCase=0.02 ,   lowerCamelCase=1e-12 ,   lowerCamelCase=1 ,   lowerCamelCase=0 ,   lowerCamelCase=2 ,   lowerCamelCase="absolute" ,   lowerCamelCase=True ,   lowerCamelCase=None ,   lowerCamelCase=False ,   lowerCamelCase=2 ,   lowerCamelCase=False ,   lowerCamelCase=True ,   lowerCamelCase=True ,   lowerCamelCase=("en_XX",) ,   lowerCamelCase=None ,   **lowerCamelCase ,   )							->Any:



         '''simple docstring'''
         super().__init__(pad_token_id=lowerCamelCase ,   bos_token_id=lowerCamelCase ,   eos_token_id=lowerCamelCase ,   **lowerCamelCase      )

         __a				     =    vocab_size
         __a				     =    hidden_size
         __a				     =    num_hidden_layers
         __a				     =    num_attention_heads
         __a				     =    hidden_act
         __a				     =    intermediate_size
         __a				     =    hidden_dropout_prob
         __a				     =    attention_probs_dropout_prob
         __a				     =    max_position_embeddings
         __a				     =    type_vocab_size
         __a				     =    initializer_range
         __a				     =    layer_norm_eps
         __a				     =    position_embedding_type
         __a				     =    use_cache
         __a				     =    classifier_dropout
         __a				     =    pre_norm
         __a				     =    adapter_reduction_factor
         __a				     =    adapter_layer_norm
         __a				     =    adapter_reuse_layer_norm
         __a				     =    ln_before_adapter
         __a				     =    list(lowerCamelCase      )
         __a				     =    default_language







class 							__SCREAMING_SNAKE_CASE       (							_lowerCAmelCase     ):




      @property
      def  __UpperCamelCase							(  self      )							->Mapping[str, Mapping[int, str]]:



         '''simple docstring'''
         if self.task == "multiple-choice":
            __a				     =    {0: 'batch', 1: 'choice', 2: 'sequence'}
         else:
            __a				     =    {0: 'batch', 1: 'sequence'}
         return OrderedDict(
             [
                 ('input_ids', dynamic_axis),
                 ('attention_mask', dynamic_axis),
             ]      )