

'''simple docstring'''





import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
							from .tokenization_xlnet import XLNetTokenizer
else:
							UpperCamelCase_							=					None


UpperCamelCase_							=					logging.get_logger(__name__)

UpperCamelCase_							=					{"""vocab_file""": """spiece.model""", """tokenizer_file""": """tokenizer.json"""}

UpperCamelCase_							=					{
    """vocab_file""": {
        """xlnet-base-cased""": """https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model""",
        """xlnet-large-cased""": """https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model""",
    },
    """tokenizer_file""": {
        """xlnet-base-cased""": """https://huggingface.co/xlnet-base-cased/resolve/main/tokenizer.json""",
        """xlnet-large-cased""": """https://huggingface.co/xlnet-large-cased/resolve/main/tokenizer.json""",
    },
}

UpperCamelCase_							=					{
    """xlnet-base-cased""": None,
    """xlnet-large-cased""": None,
}

UpperCamelCase_							=					"""▁"""

# Segments (not really needed)
UpperCamelCase_							=					0
UpperCamelCase_							=					1
UpperCamelCase_							=					2
UpperCamelCase_							=					3
UpperCamelCase_							=					4





class 		__SCREAMING_SNAKE_CASE       ( lowercase__			):
				lowerCamelCase_							=						VOCAB_FILES_NAMES
				lowerCamelCase_							=						PRETRAINED_VOCAB_FILES_MAP
				lowerCamelCase_							=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				lowerCamelCase_							=						'left'
				lowerCamelCase_							=						XLNetTokenizer







				def __init__(					self    :		Dict		,				UpperCAmelCase__    :		List[str]=None		,				UpperCAmelCase__    :		Dict=None		,				UpperCAmelCase__    :		int=False		,				UpperCAmelCase__    :		Optional[int]=True		,				UpperCAmelCase__    :		List[Any]=False		,				UpperCAmelCase__    :		Any="<s>"		,				UpperCAmelCase__    :		Union[str, Any]="</s>"		,				UpperCAmelCase__    :		Dict="<unk>"		,				UpperCAmelCase__    :		Optional[Any]="<sep>"		,				UpperCAmelCase__    :		Union[str, Any]="<pad>"		,				UpperCAmelCase__    :		Optional[Any]="<cls>"		,				UpperCAmelCase__    :		Optional[int]="<mask>"		,				UpperCAmelCase__    :		Dict=["<eop>", "<eod>"]		,				**UpperCAmelCase__    :		Union[str, Any]		,				):
										'''simple docstring'''
										# Mask token behave like a normal word, i.e. include the space before it
										lowercase   :       List[Any]								=AddedToken(UpperCAmelCase__		,				lstrip=UpperCAmelCase__		,				rstrip=UpperCAmelCase__      ) if isinstance(UpperCAmelCase__		,				UpperCAmelCase__      ) else mask_token

										super().__init__(
										    vocab_file=UpperCAmelCase__		,				tokenizer_file=UpperCAmelCase__		,				do_lower_case=UpperCAmelCase__		,				remove_space=UpperCAmelCase__		,				keep_accents=UpperCAmelCase__		,				bos_token=UpperCAmelCase__		,				eos_token=UpperCAmelCase__		,				unk_token=UpperCAmelCase__		,				sep_token=UpperCAmelCase__		,				pad_token=UpperCAmelCase__		,				cls_token=UpperCAmelCase__		,				mask_token=UpperCAmelCase__		,				additional_special_tokens=UpperCAmelCase__		,				**UpperCAmelCase__		,				)

										lowercase   :       List[Any]								=3
										lowercase   :       Tuple								=do_lower_case
										lowercase   :       Tuple								=remove_space
										lowercase   :       Optional[Any]								=keep_accents
										lowercase   :       Any								=vocab_file
										lowercase   :       Optional[Any]								=False if not self.vocab_file else True







				def    lowerCamelCase_  (					self    :		Any		,				UpperCAmelCase__    :		List[int]		,				UpperCAmelCase__    :		Optional[List[int]] = None      ):
										'''simple docstring'''
										lowercase   :       Optional[Any]								=[self.sep_token_id]
										lowercase   :       Union[str, Any]								=[self.cls_token_id]
										if token_ids_a is None:
																return token_ids_a + sep + cls
										return token_ids_a + sep + token_ids_a + sep + cls







				def    lowerCamelCase_  (					self    :		str		,				UpperCAmelCase__    :		List[int]		,				UpperCAmelCase__    :		Optional[List[int]] = None      ):
										'''simple docstring'''
										lowercase   :       str								=[self.sep_token_id]
										lowercase   :       Any								=[2]

										if token_ids_a is None:
																return len(token_ids_a + sep      ) * [0] + cls_segment_id
										return len(token_ids_a + sep      ) * [0] + len(token_ids_a + sep      ) * [1] + cls_segment_id







				def    lowerCamelCase_  (					self    :		List[Any]		,				UpperCAmelCase__    :		str		,				UpperCAmelCase__    :		Optional[str] = None      ):
										'''simple docstring'''
										if not self.can_save_slow_tokenizer:
																raise ValueError(
																    '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
																    '''tokenizer.'''      )

										if not os.path.isdir(UpperCAmelCase__      ):
																logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''      )
																return
										lowercase   :       List[Any]								=os.path.join(
										    UpperCAmelCase__		,				(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']      )

										if os.path.abspath(self.vocab_file      ) != os.path.abspath(UpperCAmelCase__      ):
																copyfile(self.vocab_file		,				UpperCAmelCase__      )

										return (out_vocab_file,)








'''simple docstring'''





# coding=utf-8
# Copyright 2023 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# this script dumps information about the environment

import os
import platform
import sys


UpperCamelCase_							=					"""3"""

print("""Python version:""", sys.version)

print("""OS platform:""", platform.platform())
print("""OS architecture:""", platform.machine())

try:
							import torch

							print("""Torch version:""", torch.__version__)
							print("""Cuda available:""", torch.cuda.is_available())
							print("""Cuda version:""", torch.version.cuda)
							print("""CuDNN version:""", torch.backends.cudnn.version())
							print("""Number of GPUs available:""", torch.cuda.device_count())
except ImportError:
							print("""Torch version:""", None)

try:
							import transformers

							print("""transformers version:""", transformers.__version__)
except ImportError:
							print("""transformers version:""", None)





