
"""simple docstring"""



import json
from typing import TYPE_CHECKING, List, Optional, Tuple

from tokenizers import pre_tokenizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_gpta import GPTaTokenizer


if TYPE_CHECKING:
 from transformers.pipelines.conversational import Conversation


a       :						List[Any]       		=		logging.get_logger(__name__)

a       :						Tuple       		=		{'''vocab_file''': '''vocab.json''', '''merges_file''': '''merges.txt''', '''tokenizer_file''': '''tokenizer.json'''}

a       :						Optional[int]       		=		{
    '''vocab_file''': {
        '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/vocab.json''',
        '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/vocab.json''',
        '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/vocab.json''',
        '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/vocab.json''',
        '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/vocab.json''',
    },
    '''merges_file''': {
        '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/merges.txt''',
        '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/merges.txt''',
        '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/merges.txt''',
        '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/merges.txt''',
        '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/merges.txt''',
    },
    '''tokenizer_file''': {
        '''gpt2''': '''https://huggingface.co/gpt2/resolve/main/tokenizer.json''',
        '''gpt2-medium''': '''https://huggingface.co/gpt2-medium/resolve/main/tokenizer.json''',
        '''gpt2-large''': '''https://huggingface.co/gpt2-large/resolve/main/tokenizer.json''',
        '''gpt2-xl''': '''https://huggingface.co/gpt2-xl/resolve/main/tokenizer.json''',
        '''distilgpt2''': '''https://huggingface.co/distilgpt2/resolve/main/tokenizer.json''',
    },
}

a       :						Tuple       		=		{
    '''gpt2''': 1_0_2_4,
    '''gpt2-medium''': 1_0_2_4,
    '''gpt2-large''': 1_0_2_4,
    '''gpt2-xl''': 1_0_2_4,
    '''distilgpt2''': 1_0_2_4,
}




class 		a_    (			_UpperCAmelCase						):
    a   :					Optional[int]			     =					VOCAB_FILES_NAMES
    a   :					Optional[Any]			     =					PRETRAINED_VOCAB_FILES_MAP
    a   :					Tuple			     =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    a   :					Optional[Any]			     =					['input_ids', 'attention_mask']
    a   :					str			     =					GPTaTokenizer







    def __init__(      self      :							Optional[Any]  ,       __UpperCamelCase      :							Any=None  ,       __UpperCamelCase      :							List[str]=None  ,       __UpperCamelCase      :							int=None  ,       __UpperCamelCase      :							List[str]="<|endoftext|>"  ,       __UpperCamelCase      :							Union[str, Any]="<|endoftext|>"  ,       __UpperCamelCase      :							str="<|endoftext|>"  ,       __UpperCamelCase      :							Any=False  ,       **__UpperCamelCase      :							List[str]  ,       )		->Optional[int]:






        '''simple docstring'''






        super().__init__(
            __UpperCamelCase  ,       __UpperCamelCase  ,       tokenizer_file=__UpperCamelCase  ,       unk_token=__UpperCamelCase  ,       bos_token=__UpperCamelCase  ,       eos_token=__UpperCamelCase  ,       add_prefix_space=__UpperCamelCase  ,       **__UpperCamelCase  ,       )

        _UpperCAmelCase        = kwargs.pop("""add_bos_token"""  ,       __UpperCamelCase  )

        _UpperCAmelCase        = json.loads(self.backend_tokenizer.pre_tokenizer.__getstate__()  )
        if pre_tok_state.get("""add_prefix_space"""  ,       __UpperCamelCase  ) != add_prefix_space:
            _UpperCAmelCase        = getattr(__UpperCamelCase  ,       pre_tok_state.pop("""type"""  )  )
            _UpperCAmelCase        = add_prefix_space
            _UpperCAmelCase        = pre_tok_class(**__UpperCamelCase  )

        _UpperCAmelCase        = add_prefix_space







    def   _snake_case       (      self      :							Any  ,       *__UpperCamelCase      :							Optional[int]  ,       **__UpperCamelCase      :							str  )		->BatchEncoding:






        '''simple docstring'''






        _UpperCAmelCase        = kwargs.get("""is_split_into_words"""  ,       __UpperCamelCase  )
        assert self.add_prefix_space or not is_split_into_words, (
            f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
            "to use it with pretokenized inputs."
        )

        return super()._batch_encode_plus(*__UpperCamelCase  ,       **__UpperCamelCase  )







    def   _snake_case       (      self      :							List[Any]  ,       *__UpperCamelCase      :							Dict  ,       **__UpperCamelCase      :							Any  )		->BatchEncoding:






        '''simple docstring'''






        _UpperCAmelCase        = kwargs.get("""is_split_into_words"""  ,       __UpperCamelCase  )

        assert self.add_prefix_space or not is_split_into_words, (
            f"""You need to instantiate {self.__class__.__name__} with add_prefix_space=True """
            "to use it with pretokenized inputs."
        )

        return super()._encode_plus(*__UpperCamelCase  ,       **__UpperCamelCase  )







    def   _snake_case       (      self      :							Tuple  ,       __UpperCamelCase      :							str  ,       __UpperCamelCase      :							Optional[str] = None  )		->Tuple[str]:






        '''simple docstring'''






        _UpperCAmelCase        = self._tokenizer.model.save(__UpperCamelCase  ,       name=__UpperCamelCase  )
        return tuple(__UpperCamelCase  )







    def   _snake_case       (      self      :							Union[str, Any]  ,       __UpperCamelCase      :							"Conversation"  )		->List[int]:






        '''simple docstring'''






        _UpperCAmelCase        = []
        for is_user, text in conversation.iter_texts():
            input_ids.extend(self.encode(__UpperCamelCase  ,       add_special_tokens=__UpperCamelCase  ) + [self.eos_token_id]  )

        if len(__UpperCamelCase  ) > self.model_max_length:
            _UpperCAmelCase        = input_ids[-self.model_max_length :]
        return input_ids

"""simple docstring"""



import importlib
import os
from dataclasses import dataclass
from enum import Enum
from typing import Any, Dict, Optional, Union

import torch

from ..utils import BaseOutput


a       :						int       		=		'''scheduler_config.json'''




class 		a_    (			_UpperCAmelCase						):
    a   :					List[Any]			     =					1
    a   :					Tuple			     =					2
    a   :					Dict			     =					3
    a   :					str			     =					4
    a   :					Optional[int]			     =					5
    a   :					Any			     =					6
    a   :					int			     =					7
    a   :					Any			     =					8
    a   :					List[Any]			     =					9
    a   :					Any			     =					10
    a   :					List[str]			     =					11
    a   :					Optional[int]			     =					12
    a   :					Any			     =					13
    a   :					Dict			     =					14







@dataclass
class 		a_    (			_UpperCAmelCase						):
    a   :					torch.FloatTensor







class 		a_    :
    a   :					Dict			     =					SCHEDULER_CONFIG_NAME
    a   :					List[str]			     =					[]
    a   :					Optional[int]			     =					True







    @classmethod
    def   _snake_case       (      cls      :							List[str]  ,       __UpperCamelCase      :							Dict[str, Any] = None  ,       __UpperCamelCase      :							Optional[str] = None  ,       __UpperCamelCase      :							List[str]=False  ,       **__UpperCamelCase      :							int  ,       )		->Dict:






        '''simple docstring'''






        _UpperCAmelCase     ,_UpperCAmelCase     ,_UpperCAmelCase        = cls.load_config(
            pretrained_model_name_or_path=__UpperCamelCase  ,       subfolder=__UpperCamelCase  ,       return_unused_kwargs=__UpperCamelCase  ,       return_commit_hash=__UpperCamelCase  ,       **__UpperCamelCase  ,       )
        return cls.from_config(__UpperCamelCase  ,       return_unused_kwargs=__UpperCamelCase  ,       **__UpperCamelCase  )







    def   _snake_case       (      self      :							Tuple  ,       __UpperCamelCase      :							Union[str, os.PathLike]  ,       __UpperCamelCase      :							bool = False  ,       **__UpperCamelCase      :							List[str]  )		->Any:






        '''simple docstring'''






        self.save_config(save_directory=__UpperCamelCase  ,       push_to_hub=__UpperCamelCase  ,       **__UpperCamelCase  )







    @property
    def   _snake_case       (      self      :							Optional[int]  )		->List[Any]:






        '''simple docstring'''






        return self._get_compatibles()







    @classmethod
    def   _snake_case       (      cls      :							List[str]  )		->Dict:






        '''simple docstring'''






        _UpperCAmelCase        = list(set([cls.__name__] + cls._compatibles  )  )
        _UpperCAmelCase        = importlib.import_module(__name__.split("""."""  )[0]  )
        _UpperCAmelCase        = [
            getattr(__UpperCamelCase  ,       __UpperCamelCase  ) for c in compatible_classes_str if hasattr(__UpperCamelCase  ,       __UpperCamelCase  )
        ]
        return compatible_classes