



'''simple docstring'''




import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
						from .tokenization_xlnet import XLNetTokenizer
else:
						UpperCamelCase__ :			List[Any]												=				None


UpperCamelCase__ :			Union[str, Any]												=				logging.get_logger(__name__)

UpperCamelCase__ :			str												=				{'''vocab_file''': '''spiece.model''', '''tokenizer_file''': '''tokenizer.json'''}

UpperCamelCase__ :			Any												=				{
    '''vocab_file''': {
        '''xlnet-base-cased''': '''https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model''',
        '''xlnet-large-cased''': '''https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model''',
    },
    '''tokenizer_file''': {
        '''xlnet-base-cased''': '''https://huggingface.co/xlnet-base-cased/resolve/main/tokenizer.json''',
        '''xlnet-large-cased''': '''https://huggingface.co/xlnet-large-cased/resolve/main/tokenizer.json''',
    },
}

UpperCamelCase__ :			Optional[Any]												=				{
    '''xlnet-base-cased''': None,
    '''xlnet-large-cased''': None,
}

UpperCamelCase__ :			Optional[Any]												=				'''▁'''

# Segments (not really needed)
UpperCamelCase__ :			Optional[int]												=				0
UpperCamelCase__ :			str												=				1
UpperCamelCase__ :			List[Any]												=				2
UpperCamelCase__ :			Union[str, Any]												=				3
UpperCamelCase__ :			List[Any]												=				4






class 		_UpperCamelCase						(							lowerCamelCase__					):


			'''simple docstring'''

			_A   :	Union[str, Any]    					=	VOCAB_FILES_NAMES
			_A   :	Union[str, Any]    					=	PRETRAINED_VOCAB_FILES_MAP
			_A   :	Dict    					=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			_A   :	List[Any]    					=	'''left'''
			_A   :	List[Any]    					=	XLNetTokenizer

			def __init__(							self  :				Union[str, Any]							,  lowerCAmelCase__  :				Any=None							,  lowerCAmelCase__  :				int=None							,  lowerCAmelCase__  :				int=False							,  lowerCAmelCase__  :				Dict=True							,  lowerCAmelCase__  :				Tuple=False							,  lowerCAmelCase__  :				str="<s>"							,  lowerCAmelCase__  :				Optional[Any]="</s>"							,  lowerCAmelCase__  :				Union[str, Any]="<unk>"							,  lowerCAmelCase__  :				Optional[int]="<sep>"							,  lowerCAmelCase__  :				Optional[Any]="<pad>"							,  lowerCAmelCase__  :				int="<cls>"							,  lowerCAmelCase__  :				Optional[int]="<mask>"							,  lowerCAmelCase__  :				Optional[Any]=["<eop>", "<eod>"]							,  **lowerCAmelCase__  :				List[str]							,  ):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE :	Any					 =	AddedToken(lowerCAmelCase__							,  lstrip=lowerCAmelCase__							,  rstrip=lowerCAmelCase__		) if isinstance(lowerCAmelCase__							,  lowerCAmelCase__		) else mask_token

					super().__init__(
					    vocab_file=lowerCAmelCase__							,  tokenizer_file=lowerCAmelCase__							,  do_lower_case=lowerCAmelCase__							,  remove_space=lowerCAmelCase__							,  keep_accents=lowerCAmelCase__							,  bos_token=lowerCAmelCase__							,  eos_token=lowerCAmelCase__							,  unk_token=lowerCAmelCase__							,  sep_token=lowerCAmelCase__							,  pad_token=lowerCAmelCase__							,  cls_token=lowerCAmelCase__							,  mask_token=lowerCAmelCase__							,  additional_special_tokens=lowerCAmelCase__							,  **lowerCAmelCase__							,  )

					__SCREAMING_SNAKE_CASE :	Dict					 =	3
					__SCREAMING_SNAKE_CASE :	Tuple					 =	do_lower_case
					__SCREAMING_SNAKE_CASE :	Union[str, Any]					 =	remove_space
					__SCREAMING_SNAKE_CASE :	Dict					 =	keep_accents
					__SCREAMING_SNAKE_CASE :	str					 =	vocab_file
					__SCREAMING_SNAKE_CASE :	str					 =	False if not self.vocab_file else True

			def UpperCamelCase__      (							self  :				str							,  lowerCAmelCase__  :				List[int]							,  lowerCAmelCase__  :				Optional[List[int]] = None		):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE :	Dict					 =	[self.sep_token_id]
					__SCREAMING_SNAKE_CASE :	List[Any]					 =	[self.cls_token_id]
					if token_ids_a is None:
							return token_ids_a + sep + cls
					return token_ids_a + sep + token_ids_a + sep + cls

			def UpperCamelCase__      (							self  :				List[Any]							,  lowerCAmelCase__  :				List[int]							,  lowerCAmelCase__  :				Optional[List[int]] = None		):





					"""simple docstring"""






					__SCREAMING_SNAKE_CASE :	Optional[Any]					 =	[self.sep_token_id]
					__SCREAMING_SNAKE_CASE :	str					 =	[2]

					if token_ids_a is None:
							return len(token_ids_a + sep		) * [0] + cls_segment_id
					return len(token_ids_a + sep		) * [0] + len(token_ids_a + sep		) * [1] + cls_segment_id



			def UpperCamelCase__      (							self  :				int							,  lowerCAmelCase__  :				str							,  lowerCAmelCase__  :				Optional[str] = None		):





					"""simple docstring"""






					if not self.can_save_slow_tokenizer:
							raise ValueError(
							    """Your fast tokenizer does not have the necessary information to save the vocabulary for a slow """
							    """tokenizer."""		)

					if not os.path.isdir(lowerCAmelCase__		):
							logger.error(F"Vocabulary path ({save_directory}) should be a directory"		)
							return
					__SCREAMING_SNAKE_CASE :	Optional[Any]					 =	os.path.join(
					    lowerCAmelCase__							,  (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]		)

					if os.path.abspath(self.vocab_file		) != os.path.abspath(lowerCAmelCase__		):
							copyfile(self.vocab_file							,  lowerCAmelCase__		)

					return (out_vocab_file,)




'''simple docstring'''




from ..utils import DummyObject, requires_backends






class 		_UpperCamelCase						(							metaclass=lowerCamelCase__					):


			'''simple docstring'''

			_A   :	Union[str, Any]    					=	['''torch''', '''transformers''', '''onnx''']

			def __init__(							self  :				Optional[int]							,  *lowerCAmelCase__  :				Optional[Any]							,  **lowerCAmelCase__  :				int		):





					"""simple docstring"""






					requires_backends(self							,  ["""torch""", """transformers""", """onnx"""]		)

			@classmethod
			def UpperCamelCase__      (							cls  :				Tuple							,  *lowerCAmelCase__  :				Union[str, Any]							,  **lowerCAmelCase__  :				Optional[int]		):





					"""simple docstring"""






					requires_backends(cls							,  ["""torch""", """transformers""", """onnx"""]		)



			@classmethod
			def UpperCamelCase__      (							cls  :				str							,  *lowerCAmelCase__  :				int							,  **lowerCAmelCase__  :				Optional[Any]		):





					"""simple docstring"""






					requires_backends(cls							,  ["""torch""", """transformers""", """onnx"""]		)






class 		_UpperCamelCase						(							metaclass=lowerCamelCase__					):


			'''simple docstring'''

			_A   :	Dict    					=	['''torch''', '''transformers''', '''onnx''']

			def __init__(							self  :				str							,  *lowerCAmelCase__  :				str							,  **lowerCAmelCase__  :				List[str]		):





					"""simple docstring"""






					requires_backends(self							,  ["""torch""", """transformers""", """onnx"""]		)

			@classmethod
			def UpperCamelCase__      (							cls  :				Optional[Any]							,  *lowerCAmelCase__  :				Dict							,  **lowerCAmelCase__  :				Dict		):





					"""simple docstring"""






					requires_backends(cls							,  ["""torch""", """transformers""", """onnx"""]		)



			@classmethod
			def UpperCamelCase__      (							cls  :				str							,  *lowerCAmelCase__  :				List[str]							,  **lowerCAmelCase__  :				str		):





					"""simple docstring"""






					requires_backends(cls							,  ["""torch""", """transformers""", """onnx"""]		)






class 		_UpperCamelCase						(							metaclass=lowerCamelCase__					):


			'''simple docstring'''

			_A   :	Tuple    					=	['''torch''', '''transformers''', '''onnx''']

			def __init__(							self  :				str							,  *lowerCAmelCase__  :				List[str]							,  **lowerCAmelCase__  :				Dict		):





					"""simple docstring"""






					requires_backends(self							,  ["""torch""", """transformers""", """onnx"""]		)

			@classmethod
			def UpperCamelCase__      (							cls  :				Any							,  *lowerCAmelCase__  :				Tuple							,  **lowerCAmelCase__  :				Any		):





					"""simple docstring"""






					requires_backends(cls							,  ["""torch""", """transformers""", """onnx"""]		)



			@classmethod
			def UpperCamelCase__      (							cls  :				List[str]							,  *lowerCAmelCase__  :				Union[str, Any]							,  **lowerCAmelCase__  :				Dict		):





					"""simple docstring"""






					requires_backends(cls							,  ["""torch""", """transformers""", """onnx"""]		)






class 		_UpperCamelCase						(							metaclass=lowerCamelCase__					):


			'''simple docstring'''

			_A   :	Optional[int]    					=	['''torch''', '''transformers''', '''onnx''']

			def __init__(							self  :				Optional[int]							,  *lowerCAmelCase__  :				Dict							,  **lowerCAmelCase__  :				List[Any]		):





					"""simple docstring"""






					requires_backends(self							,  ["""torch""", """transformers""", """onnx"""]		)

			@classmethod
			def UpperCamelCase__      (							cls  :				Any							,  *lowerCAmelCase__  :				List[Any]							,  **lowerCAmelCase__  :				List[str]		):





					"""simple docstring"""






					requires_backends(cls							,  ["""torch""", """transformers""", """onnx"""]		)



			@classmethod
			def UpperCamelCase__      (							cls  :				List[str]							,  *lowerCAmelCase__  :				str							,  **lowerCAmelCase__  :				Optional[int]		):





					"""simple docstring"""






					requires_backends(cls							,  ["""torch""", """transformers""", """onnx"""]		)






class 		_UpperCamelCase						(							metaclass=lowerCamelCase__					):


			'''simple docstring'''

			_A   :	Union[str, Any]    					=	['''torch''', '''transformers''', '''onnx''']

			def __init__(							self  :				Optional[int]							,  *lowerCAmelCase__  :				int							,  **lowerCAmelCase__  :				List[str]		):





					"""simple docstring"""






					requires_backends(self							,  ["""torch""", """transformers""", """onnx"""]		)

			@classmethod
			def UpperCamelCase__      (							cls  :				List[str]							,  *lowerCAmelCase__  :				int							,  **lowerCAmelCase__  :				int		):





					"""simple docstring"""






					requires_backends(cls							,  ["""torch""", """transformers""", """onnx"""]		)



			@classmethod
			def UpperCamelCase__      (							cls  :				str							,  *lowerCAmelCase__  :				int							,  **lowerCAmelCase__  :				List[Any]		):





					"""simple docstring"""






					requires_backends(cls							,  ["""torch""", """transformers""", """onnx"""]		)






class 		_UpperCamelCase						(							metaclass=lowerCamelCase__					):


			'''simple docstring'''

			_A   :	Optional[Any]    					=	['''torch''', '''transformers''', '''onnx''']

			def __init__(							self  :				Optional[int]							,  *lowerCAmelCase__  :				Tuple							,  **lowerCAmelCase__  :				List[str]		):





					"""simple docstring"""






					requires_backends(self							,  ["""torch""", """transformers""", """onnx"""]		)

			@classmethod
			def UpperCamelCase__      (							cls  :				Tuple							,  *lowerCAmelCase__  :				Dict							,  **lowerCAmelCase__  :				List[Any]		):





					"""simple docstring"""






					requires_backends(cls							,  ["""torch""", """transformers""", """onnx"""]		)



			@classmethod
			def UpperCamelCase__      (							cls  :				List[str]							,  *lowerCAmelCase__  :				Dict							,  **lowerCAmelCase__  :				Tuple		):





					"""simple docstring"""






					requires_backends(cls							,  ["""torch""", """transformers""", """onnx"""]		)