


"""simple docstring"""






import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
   from .tokenization_xlnet import XLNetTokenizer
else:
   __snake_case		=       None


__snake_case		=       logging.get_logger(__name__)

__snake_case		=       {'''vocab_file''': '''spiece.model''', '''tokenizer_file''': '''tokenizer.json'''}

__snake_case		=       {
    '''vocab_file''': {
        '''xlnet-base-cased''': '''https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model''',
        '''xlnet-large-cased''': '''https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model''',
    },
    '''tokenizer_file''': {
        '''xlnet-base-cased''': '''https://huggingface.co/xlnet-base-cased/resolve/main/tokenizer.json''',
        '''xlnet-large-cased''': '''https://huggingface.co/xlnet-large-cased/resolve/main/tokenizer.json''',
    },
}

__snake_case		=       {
    '''xlnet-base-cased''': None,
    '''xlnet-large-cased''': None,
}

__snake_case		=       '''▁'''

# Segments (not really needed)
__snake_case		=       0
__snake_case		=       1
__snake_case		=       2
__snake_case		=       3
__snake_case		=       4


class     __lowerCamelCase  (					a__				):

     '''simple docstring'''
     A_       :	Optional[int]    =		VOCAB_FILES_NAMES
     A_       :	Union[str, Any]    =		PRETRAINED_VOCAB_FILES_MAP
     A_       :	int    =		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     A_       :	Optional[Any]    =		'left'
     A_       :	int    =		XLNetTokenizer


     def __init__(      self    ,				__UpperCAmelCase=None    ,				__UpperCAmelCase=None    ,				__UpperCAmelCase=False    ,				__UpperCAmelCase=True    ,				__UpperCAmelCase=False    ,				__UpperCAmelCase="<s>"    ,				__UpperCAmelCase="</s>"    ,				__UpperCAmelCase="<unk>"    ,				__UpperCAmelCase="<sep>"    ,				__UpperCAmelCase="<pad>"    ,				__UpperCAmelCase="<cls>"    ,				__UpperCAmelCase="<mask>"    ,				__UpperCAmelCase=["<eop>", "<eod>"]    ,				**__UpperCAmelCase    ,				)       ->							Dict:
          # Mask token behave like a normal word, i.e. include the space before it
          _a    	=				AddedToken(__UpperCAmelCase    ,				lstrip=__UpperCAmelCase    ,				rstrip=__UpperCAmelCase							) if isinstance(__UpperCAmelCase    ,				__UpperCAmelCase							) else mask_token

          super().__init__(
              vocab_file=__UpperCAmelCase    ,				tokenizer_file=__UpperCAmelCase    ,				do_lower_case=__UpperCAmelCase    ,				remove_space=__UpperCAmelCase    ,				keep_accents=__UpperCAmelCase    ,				bos_token=__UpperCAmelCase    ,				eos_token=__UpperCAmelCase    ,				unk_token=__UpperCAmelCase    ,				sep_token=__UpperCAmelCase    ,				pad_token=__UpperCAmelCase    ,				cls_token=__UpperCAmelCase    ,				mask_token=__UpperCAmelCase    ,				additional_special_tokens=__UpperCAmelCase    ,				**__UpperCAmelCase    ,				)

          _a    	=				3
          _a    	=				do_lower_case
          _a    	=				remove_space
          _a    	=				keep_accents
          _a    	=				vocab_file
          _a    	=				False if not self.vocab_file else True


     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase    ,				__UpperCAmelCase = None							)       ->							List[int]:
          _a    	=				[self.sep_token_id]
          _a    	=				[self.cls_token_id]
          if token_ids_a is None:
               return token_ids_a + sep + cls
          return token_ids_a + sep + token_ids_a + sep + cls


     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase    ,				__UpperCAmelCase = None							)       ->							List[int]:
          _a    	=				[self.sep_token_id]
          _a    	=				[2]

          if token_ids_a is None:
               return len(token_ids_a + sep							) * [0] + cls_segment_id
          return len(token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1] + cls_segment_id





     def 							_UpperCAmelCase      (      self    ,				__UpperCAmelCase    ,				__UpperCAmelCase = None							)       ->							Tuple[str]:
          if not self.can_save_slow_tokenizer:
               raise ValueError(
                   '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
                   '''tokenizer.'''							)

          if not os.path.isdir(__UpperCAmelCase							):
               logger.error(F'Vocabulary path ({save_directory}) should be a directory'							)
               return
          _a    	=				os.path.join(
              __UpperCAmelCase    ,				(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']							)

          if os.path.abspath(self.vocab_file							) != os.path.abspath(__UpperCAmelCase							):
               copyfile(self.vocab_file    ,				__UpperCAmelCase							)

          return (out_vocab_file,)



"""simple docstring"""






from typing import Any
def       A_   (   _lowerCAmelCase     :					list       ):






     """simple docstring"""

     if not input_list:
          return []
     _a    	=				[input_list.count(_lowerCAmelCase       ) for value in input_list]
     _a    	=				max(_lowerCAmelCase       )  # Gets the maximum count in the input list.
     # Gets values of modes
     return sorted({input_list[i] for i, value in enumerate(_lowerCAmelCase       ) if value == y}       )


if __name__ == "__main__":
   import doctest

   doctest.testmod()