





'''simple docstring'''






from collections import OrderedDict
from typing import TYPE_CHECKING, Any, Mapping, Optional

from ...configuration_utils import PretrainedConfig
from ...onnx import OnnxConfig
from ...utils import logging


if TYPE_CHECKING:
      from ... import FeatureExtractionMixin, TensorType

A__							:  Tuple    =logging.get_logger(__name__)

A__							:  Any    ={
    '''openai/imagegpt-small''': '''''',
    '''openai/imagegpt-medium''': '''''',
    '''openai/imagegpt-large''': '''''',
}




class 					UpperCAmelCase	(     snake_case_  ):
      _lowercase:     Tuple			=  '''imagegpt'''
      _lowercase:     Optional[int]			=  ['''past_key_values''']
      _lowercase:     str			=  {
          '''hidden_size''': '''n_embd''',
          '''max_position_embeddings''': '''n_positions''',
          '''num_attention_heads''': '''n_head''',
          '''num_hidden_layers''': '''n_layer''',
      }




      def __init__(     self     :					Optional[int]     ,     __snake_case     :					Optional[Any]=5_12 + 1     ,     __snake_case     :					Union[str, Any]=32 * 32     ,     __snake_case     :					List[Any]=5_12     ,     __snake_case     :					Any=24     ,     __snake_case     :					Optional[Any]=8     ,     __snake_case     :					List[Any]=None     ,     __snake_case     :					str="quick_gelu"     ,     __snake_case     :					List[Any]=0.1     ,     __snake_case     :					str=0.1     ,     __snake_case     :					Optional[int]=0.1     ,     __snake_case     :					Tuple=1E-5     ,     __snake_case     :					Optional[int]=0.02     ,     __snake_case     :					str=True     ,     __snake_case     :					Dict=True     ,     __snake_case     :					Union[str, Any]=False     ,     __snake_case     :					Dict=False     ,     __snake_case     :					Union[str, Any]=False     ,     **__snake_case     :					Union[str, Any]     ,     )							-> List[Any]:
            _lowerCAmelCase					=				vocab_size
            _lowerCAmelCase					=				n_positions
            _lowerCAmelCase					=				n_embd
            _lowerCAmelCase					=				n_layer
            _lowerCAmelCase					=				n_head
            _lowerCAmelCase					=				n_inner
            _lowerCAmelCase					=				activation_function
            _lowerCAmelCase					=				resid_pdrop
            _lowerCAmelCase					=				embd_pdrop
            _lowerCAmelCase					=				attn_pdrop
            _lowerCAmelCase					=				layer_norm_epsilon
            _lowerCAmelCase					=				initializer_range
            _lowerCAmelCase					=				scale_attn_weights
            _lowerCAmelCase					=				use_cache
            _lowerCAmelCase					=				scale_attn_by_inverse_layer_idx
            _lowerCAmelCase					=				reorder_and_upcast_attn
            _lowerCAmelCase					=				tie_word_embeddings

            super().__init__(tie_word_embeddings=__snake_case     ,     **__snake_case    )




class 					UpperCAmelCase	(     snake_case_  ):




      @property
      def 							lowercase__       (     self     :					Optional[Any]    )							-> Mapping[str, Mapping[int, str]]:
            return OrderedDict(
                [
                    ("""input_ids""", {0: """batch""", 1: """sequence"""}),
                ]    )




      def 							lowercase__       (     self     :					Union[str, Any]     ,     __snake_case     :					"FeatureExtractionMixin"     ,     __snake_case     :					int = 1     ,     __snake_case     :					int = -1     ,     __snake_case     :					bool = False     ,     __snake_case     :					Optional["TensorType"] = None     ,     __snake_case     :					int = 3     ,     __snake_case     :					int = 32     ,     __snake_case     :					int = 32     ,     )							-> Mapping[str, Any]:
            _lowerCAmelCase					=				self._generate_dummy_images(__snake_case     ,     __snake_case     ,     __snake_case     ,     __snake_case    )
            _lowerCAmelCase					=				dict(preprocessor(images=__snake_case     ,     return_tensors=__snake_case    )    )

            return inputs













'''simple docstring'''






import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
      from .tokenization_xlnet import XLNetTokenizer
else:
      A__							:  Any    =None


A__							:  Optional[int]    =logging.get_logger(__name__)

A__							:  Union[str, Any]    ={'''vocab_file''': '''spiece.model''', '''tokenizer_file''': '''tokenizer.json'''}

A__							:  List[str]    ={
    '''vocab_file''': {
        '''xlnet-base-cased''': '''https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model''',
        '''xlnet-large-cased''': '''https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model''',
    },
    '''tokenizer_file''': {
        '''xlnet-base-cased''': '''https://huggingface.co/xlnet-base-cased/resolve/main/tokenizer.json''',
        '''xlnet-large-cased''': '''https://huggingface.co/xlnet-large-cased/resolve/main/tokenizer.json''',
    },
}

A__							:  List[str]    ={
    '''xlnet-base-cased''': None,
    '''xlnet-large-cased''': None,
}

A__							:  List[str]    ='''▁'''

# Segments (not really needed)
A__							:  str    =0
A__							:  str    =1
A__							:  List[Any]    =2
A__							:  str    =3
A__							:  Optional[Any]    =4




class 					UpperCAmelCase	(     snake_case_  ):
      _lowercase:     Optional[int]			=  VOCAB_FILES_NAMES
      _lowercase:     Optional[int]			=  PRETRAINED_VOCAB_FILES_MAP
      _lowercase:     Dict			=  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
      _lowercase:     Optional[Any]			=  '''left'''
      _lowercase:     Dict			=  XLNetTokenizer




      def __init__(     self     :					List[str]     ,     __snake_case     :					Optional[Any]=None     ,     __snake_case     :					str=None     ,     __snake_case     :					Union[str, Any]=False     ,     __snake_case     :					str=True     ,     __snake_case     :					Union[str, Any]=False     ,     __snake_case     :					List[Any]="<s>"     ,     __snake_case     :					List[Any]="</s>"     ,     __snake_case     :					str="<unk>"     ,     __snake_case     :					int="<sep>"     ,     __snake_case     :					int="<pad>"     ,     __snake_case     :					Dict="<cls>"     ,     __snake_case     :					int="<mask>"     ,     __snake_case     :					Optional[int]=["<eop>", "<eod>"]     ,     **__snake_case     :					List[str]     ,     )							-> List[str]:
            # Mask token behave like a normal word, i.e. include the space before it
            _lowerCAmelCase					=				AddedToken(__snake_case     ,     lstrip=__snake_case     ,     rstrip=__snake_case    ) if isinstance(__snake_case     ,     __snake_case    ) else mask_token

            super().__init__(
                vocab_file=__snake_case     ,     tokenizer_file=__snake_case     ,     do_lower_case=__snake_case     ,     remove_space=__snake_case     ,     keep_accents=__snake_case     ,     bos_token=__snake_case     ,     eos_token=__snake_case     ,     unk_token=__snake_case     ,     sep_token=__snake_case     ,     pad_token=__snake_case     ,     cls_token=__snake_case     ,     mask_token=__snake_case     ,     additional_special_tokens=__snake_case     ,     **__snake_case     ,     )

            _lowerCAmelCase					=				3
            _lowerCAmelCase					=				do_lower_case
            _lowerCAmelCase					=				remove_space
            _lowerCAmelCase					=				keep_accents
            _lowerCAmelCase					=				vocab_file
            _lowerCAmelCase					=				False if not self.vocab_file else True




      def 							lowercase__       (     self     :					Optional[int]     ,     __snake_case     :					List[int]     ,     __snake_case     :					Optional[List[int]] = None    )							-> List[int]:
            _lowerCAmelCase					=				[self.sep_token_id]
            _lowerCAmelCase					=				[self.cls_token_id]
            if token_ids_a is None:
                  return token_ids_a + sep + cls
            return token_ids_a + sep + token_ids_a + sep + cls




      def 							lowercase__       (     self     :					str     ,     __snake_case     :					List[int]     ,     __snake_case     :					Optional[List[int]] = None    )							-> List[int]:
            _lowerCAmelCase					=				[self.sep_token_id]
            _lowerCAmelCase					=				[2]

            if token_ids_a is None:
                  return len(token_ids_a + sep    ) * [0] + cls_segment_id
            return len(token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1] + cls_segment_id




      def 							lowercase__       (     self     :					Optional[Any]     ,     __snake_case     :					str     ,     __snake_case     :					Optional[str] = None    )							-> Tuple[str]:
            if not self.can_save_slow_tokenizer:
                  raise ValueError(
                      """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
                      """tokenizer."""    )

            if not os.path.isdir(__snake_case    ):
                  logger.error(f"Vocabulary path ({save_directory}) should be a directory"    )
                  return
            _lowerCAmelCase					=				os.path.join(
                __snake_case     ,     (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]    )

            if os.path.abspath(self.vocab_file    ) != os.path.abspath(__snake_case    ):
                  copyfile(self.vocab_file     ,     __snake_case    )

            return (out_vocab_file,)






