






import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils import AddedToken
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
			from .tokenization_albert import AlbertTokenizer
else:
			UpperCAmelCase__      			=						None

UpperCAmelCase__      			=						logging.get_logger(__name__)
UpperCAmelCase__      			=						{'''vocab_file''': '''spiece.model''', '''tokenizer_file''': '''tokenizer.json'''}

UpperCAmelCase__      			=						{
    '''vocab_file''': {
        '''albert-base-v1''': '''https://huggingface.co/albert-base-v1/resolve/main/spiece.model''',
        '''albert-large-v1''': '''https://huggingface.co/albert-large-v1/resolve/main/spiece.model''',
        '''albert-xlarge-v1''': '''https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model''',
        '''albert-xxlarge-v1''': '''https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model''',
        '''albert-base-v2''': '''https://huggingface.co/albert-base-v2/resolve/main/spiece.model''',
        '''albert-large-v2''': '''https://huggingface.co/albert-large-v2/resolve/main/spiece.model''',
        '''albert-xlarge-v2''': '''https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model''',
        '''albert-xxlarge-v2''': '''https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model''',
    },
    '''tokenizer_file''': {
        '''albert-base-v1''': '''https://huggingface.co/albert-base-v1/resolve/main/tokenizer.json''',
        '''albert-large-v1''': '''https://huggingface.co/albert-large-v1/resolve/main/tokenizer.json''',
        '''albert-xlarge-v1''': '''https://huggingface.co/albert-xlarge-v1/resolve/main/tokenizer.json''',
        '''albert-xxlarge-v1''': '''https://huggingface.co/albert-xxlarge-v1/resolve/main/tokenizer.json''',
        '''albert-base-v2''': '''https://huggingface.co/albert-base-v2/resolve/main/tokenizer.json''',
        '''albert-large-v2''': '''https://huggingface.co/albert-large-v2/resolve/main/tokenizer.json''',
        '''albert-xlarge-v2''': '''https://huggingface.co/albert-xlarge-v2/resolve/main/tokenizer.json''',
        '''albert-xxlarge-v2''': '''https://huggingface.co/albert-xxlarge-v2/resolve/main/tokenizer.json''',
    },
}

UpperCAmelCase__      			=						{
    '''albert-base-v1''': 512,
    '''albert-large-v1''': 512,
    '''albert-xlarge-v1''': 512,
    '''albert-xxlarge-v1''': 512,
    '''albert-base-v2''': 512,
    '''albert-large-v2''': 512,
    '''albert-xlarge-v2''': 512,
    '''albert-xxlarge-v2''': 512,
}

UpperCAmelCase__      			=						'''▁'''




class 							snake_case_       (    __UpperCamelCase      ):

						"""simple docstring"""



						snake_case__        =      VOCAB_FILES_NAMES
						snake_case__        =      PRETRAINED_VOCAB_FILES_MAP
						snake_case__        =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						snake_case__        =      AlbertTokenizer



						def __init__(self:					Dict ,					__UpperCAmelCase:					str=None ,					__UpperCAmelCase:					Optional[Any]=None ,					__UpperCAmelCase:					Optional[int]=True ,					__UpperCAmelCase:					Any=True ,					__UpperCAmelCase:					Union[str, Any]=False ,					__UpperCAmelCase:					List[str]="[CLS]" ,					__UpperCAmelCase:					int="[SEP]" ,					__UpperCAmelCase:					int="<unk>" ,					__UpperCAmelCase:					Tuple="[SEP]" ,					__UpperCAmelCase:					Union[str, Any]="<pad>" ,					__UpperCAmelCase:					Dict="[CLS]" ,					__UpperCAmelCase:					Any="[MASK]" ,					**__UpperCAmelCase:					Any ,					)    ->     List[Any]:




											'''simple docstring'''



											__a					:						str      =					(
											    AddedToken(__UpperCAmelCase ,					lstrip=__UpperCAmelCase ,					rstrip=__UpperCAmelCase ,					normalized=__UpperCAmelCase   )
											    if isinstance(__UpperCAmelCase ,					__UpperCAmelCase   )
											    else mask_token
											)

											super().__init__(
											    __UpperCAmelCase ,					tokenizer_file=__UpperCAmelCase ,					do_lower_case=__UpperCAmelCase ,					remove_space=__UpperCAmelCase ,					keep_accents=__UpperCAmelCase ,					bos_token=__UpperCAmelCase ,					eos_token=__UpperCAmelCase ,					unk_token=__UpperCAmelCase ,					sep_token=__UpperCAmelCase ,					pad_token=__UpperCAmelCase ,					cls_token=__UpperCAmelCase ,					mask_token=__UpperCAmelCase ,					**__UpperCAmelCase ,					)

											__a					:						Any      =					do_lower_case
											__a					:						int      =					remove_space
											__a					:						int      =					keep_accents
											__a					:						str      =					vocab_file
											__a					:						Optional[Any]      =					False if not self.vocab_file else True



						def        UpperCAmelCase__					(self:					Optional[int] ,					__UpperCAmelCase:					List[int] ,					__UpperCAmelCase:					Optional[List[int]] = None   )    ->     List[int]:




											'''simple docstring'''



											__a					:						Any      =					[self.sep_token_id]
											__a					:						Union[str, Any]      =					[self.cls_token_id]
											if token_ids_a is None:
																return cls + token_ids_a + sep
											return cls + token_ids_a + sep + token_ids_a + sep



						def        UpperCAmelCase__					(self:					Dict ,					__UpperCAmelCase:					List[int] ,					__UpperCAmelCase:					Optional[List[int]] = None   )    ->     List[int]:




											'''simple docstring'''



											__a					:						int      =					[self.sep_token_id]
											__a					:						str      =					[self.cls_token_id]

											if token_ids_a is None:
																return len(cls + token_ids_a + sep   ) * [0]
											return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]



						def        UpperCAmelCase__					(self:					Dict ,					__UpperCAmelCase:					str ,					__UpperCAmelCase:					Optional[str] = None   )    ->     Tuple[str]:




											'''simple docstring'''



											if not self.can_save_slow_tokenizer:
																raise ValueError(
																    "Your fast tokenizer does not have the necessary information to save the vocabulary for a slow "
																    "tokenizer."   )

											if not os.path.isdir(__UpperCAmelCase   ):
																logger.error(f'Vocabulary path ({save_directory}) should be a directory'   )
																return
											__a					:						int      =					os.path.join(
											    __UpperCAmelCase ,					(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]   )

											if os.path.abspath(self.vocab_file   ) != os.path.abspath(__UpperCAmelCase   ):
																copyfile(self.vocab_file ,					__UpperCAmelCase   )

											return (out_vocab_file,)













# coding=utf-8
# Copyright 2020 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# this script dumps information about the environment

import os
import sys

import transformers


UpperCAmelCase__      			=						'''3'''

print('''Python version:''', sys.version)
print('''transformers version:''', transformers.__version__)

try:
			import torch

			print('''Torch version:''', torch.__version__)
			print('''Cuda available:''', torch.cuda.is_available())
			print('''Cuda version:''', torch.version.cuda)
			print('''CuDNN version:''', torch.backends.cudnn.version())
			print('''Number of GPUs available:''', torch.cuda.device_count())
			print('''NCCL version:''', torch.cuda.nccl.version())
except ImportError:
			print('''Torch version:''', None)

try:
			import deepspeed

			print('''DeepSpeed version:''', deepspeed.__version__)
except ImportError:
			print('''DeepSpeed version:''', None)

try:
			import tensorflow as tf

			print('''TensorFlow version:''', tf.__version__)
			print('''TF GPUs available:''', bool(tf.config.list_physical_devices('''GPU''')))
			print('''Number of TF GPUs available:''', len(tf.config.list_physical_devices('''GPU''')))
except ImportError:
			print('''TensorFlow version:''', None)





