



# coding=utf-8
# Copyright 2020 The HuggingFace Inc. team.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# this script dumps information about the environment

import os
import sys

import transformers


__snake_case       :	Optional[Any]            ='3'

print('Python version:', sys.version)
print('transformers version:', transformers.__version__)

try:
							import torch

							print('Torch version:', torch.__version__)
							print('Cuda available:', torch.cuda.is_available())
							print('Cuda version:', torch.version.cuda)
							print('CuDNN version:', torch.backends.cudnn.version())
							print('Number of GPUs available:', torch.cuda.device_count())
							print('NCCL version:', torch.cuda.nccl.version())
except ImportError:
							print('Torch version:', None)

try:
							import deepspeed

							print('DeepSpeed version:', deepspeed.__version__)
except ImportError:
							print('DeepSpeed version:', None)

try:
							import tensorflow as tf

							print('TensorFlow version:', tf.__version__)
							print('TF GPUs available:', bool(tf.config.list_physical_devices('GPU')))
							print('Number of TF GPUs available:', len(tf.config.list_physical_devices('GPU')))
except ImportError:
							print('TensorFlow version:', None)






import os
from shutil import copyfile
from typing import List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__snake_case       :	Union[str, Any]            =logging.get_logger(__name__)

__snake_case       :	Dict            ={'vocab_file': 'sentencepiece.model'}

__snake_case       :	Optional[Any]            ={
    'vocab_file': {
        'google/rembert': 'https://huggingface.co/google/rembert/resolve/main/sentencepiece.model',
    },
}

__snake_case       :	int            ={
    'google/rembert': 2_5_6,
}



class  lowerCamelCase__    (							lowerCamelCase__):




		'''simple docstring'''



		snake_case_			=VOCAB_FILES_NAMES
		snake_case_			=PRETRAINED_VOCAB_FILES_MAP
		snake_case_			=PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES



		def __init__(self	,__lowerCamelCase	,__lowerCamelCase=False	,__lowerCamelCase=True	,__lowerCamelCase=True	,__lowerCamelCase="[CLS]"	,__lowerCamelCase="[SEP]"	,__lowerCamelCase="[UNK]"	,__lowerCamelCase="[SEP]"	,__lowerCamelCase="[PAD]"	,__lowerCamelCase="[CLS]"	,__lowerCamelCase="[MASK]"	,**__lowerCamelCase	,)  ->    Union[str, Any]:

						"""simple docstring"""






						super().__init__(
						    do_lower_case=__lowerCamelCase	,remove_space=__lowerCamelCase	,keep_accents=__lowerCamelCase	,bos_token=__lowerCamelCase	,eos_token=__lowerCamelCase	,unk_token=__lowerCamelCase	,sep_token=__lowerCamelCase	,pad_token=__lowerCamelCase	,cls_token=__lowerCamelCase	,mask_token=__lowerCamelCase	,**__lowerCamelCase	,)

						lowerCAmelCase__  :       int				=  do_lower_case
						lowerCAmelCase__  :       Optional[Any]				=  remove_space
						lowerCAmelCase__  :       Any				=  keep_accents
						lowerCAmelCase__  :       Dict				=  vocab_file

						lowerCAmelCase__  :       List[str]				=  spm.SentencePieceProcessor()
						self.sp_model.Load(__lowerCamelCase				)



		@property
		def 	lowerCAmelCase__   (self				)  ->    List[str]:

						"""simple docstring"""






						return len(self.sp_model				)



		def 	lowerCAmelCase__   (self				)  ->    List[Any]:

						"""simple docstring"""






						lowerCAmelCase__  :       int				=  {self.convert_ids_to_tokens(__lowerCamelCase				): i for i in range(self.vocab_size				)}
						vocab.update(self.added_tokens_encoder				)
						return vocab



		def __getstate__(self				)  ->    Union[str, Any]:

						"""simple docstring"""






						lowerCAmelCase__  :       Any				=  self.__dict__.copy()
						lowerCAmelCase__  :       Dict				=  None
						return state



		def __setstate__(self	,__lowerCamelCase				)  ->    int:

						"""simple docstring"""






						lowerCAmelCase__  :       Dict				=  d
						lowerCAmelCase__  :       List[str]				=  spm.SentencePieceProcessor()
						self.sp_model.Load(self.vocab_file				)



		def 	lowerCAmelCase__   (self	,__lowerCamelCase	,__lowerCamelCase=False				)  ->    Tuple:

						"""simple docstring"""






						lowerCAmelCase__  :       str				=  self.sp_model.EncodeAsPieces(__lowerCamelCase				)
						return pieces



		def 	lowerCAmelCase__   (self	,__lowerCamelCase				)  ->    Dict:

						"""simple docstring"""






						return self.sp_model.PieceToId(__lowerCamelCase				)



		def 	lowerCAmelCase__   (self	,__lowerCamelCase				)  ->    List[str]:

						"""simple docstring"""






						return self.sp_model.IdToPiece(__lowerCamelCase				)



		def 	lowerCAmelCase__   (self	,__lowerCamelCase				)  ->    List[str]:

						"""simple docstring"""






						lowerCAmelCase__  :       Any				=  self.sp_model.decode_pieces(__lowerCamelCase				)
						return out_string



		def 	lowerCAmelCase__   (self	,__lowerCamelCase	,__lowerCamelCase = None				)  ->    List[int]:

						"""simple docstring"""






						lowerCAmelCase__  :       str				=  [self.sep_token_id]
						lowerCAmelCase__  :       int				=  [self.cls_token_id]
						if token_ids_a is None:
										return cls + token_ids_a + sep
						return cls + token_ids_a + sep + token_ids_a + sep



		def 	lowerCAmelCase__   (self	,__lowerCamelCase	,__lowerCamelCase = None	,__lowerCamelCase = False				)  ->    List[int]:

						"""simple docstring"""






						if already_has_special_tokens:
										if token_ids_a is not None:
														raise ValueError(
														    '''You should not supply a second sequence if the provided sequence of '''
														    '''ids is already formatted with special tokens for the model.'''				)
										return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

						if token_ids_a is not None:
										return [1] + ([0] * len(__lowerCamelCase				)) + [1] + ([0] * len(__lowerCamelCase				)) + [1]
						return [1] + ([0] * len(__lowerCamelCase				)) + [1]



		def 	lowerCAmelCase__   (self	,__lowerCamelCase	,__lowerCamelCase = None				)  ->    List[int]:

						"""simple docstring"""






						lowerCAmelCase__  :       Union[str, Any]				=  [self.sep_token_id]
						lowerCAmelCase__  :       Union[str, Any]				=  [self.cls_token_id]

						if token_ids_a is None:
										return len(cls + token_ids_a + sep				) * [0]
						return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]



		def 	lowerCAmelCase__   (self	,__lowerCamelCase	,__lowerCamelCase = None				)  ->    Tuple[str]:

						"""simple docstring"""






						if not os.path.isdir(__lowerCamelCase				):
										logger.error('''Vocabulary path ({}) should be a directory'''.format(__lowerCamelCase				)				)
										return
						lowerCAmelCase__  :       Union[str, Any]				=  os.path.join(
						    __lowerCamelCase	,(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']				)

						if os.path.abspath(self.vocab_file				) != os.path.abspath(__lowerCamelCase				):
										copyfile(self.vocab_file	,__lowerCamelCase				)

						return (out_vocab_file,)

