




from typing import List, Optional, Union

import numpy as np
import tensorflow as tf

from .utils import logging


__SCREAMING_SNAKE_CASE 						=logging.get_logger(__name__)

def 			a   (_lowerCAmelCase     ):
							if isinstance(_lowerCAmelCase  ,    np.ndarray     ):
														return list(tensor.shape     )

							SCREAMING_SNAKE_CASE_   	=    tf.shape(_lowerCAmelCase     )

							if tensor.shape == tf.TensorShape(_lowerCAmelCase     ):
														return dynamic

							SCREAMING_SNAKE_CASE_   	=    tensor.shape.as_list()

							return [dynamic[i] if s is None else s for i, s in enumerate(_lowerCAmelCase     )]

def 			a   (_lowerCAmelCase  ,    _lowerCAmelCase = None  ,    _lowerCAmelCase = None     ):
							return tf.nn.softmax(logits=logits + 1e-9  ,    axis=_lowerCAmelCase  ,    name=_lowerCAmelCase     )

def 			a   (_lowerCAmelCase  ,    _lowerCAmelCase  ,    _lowerCAmelCase  ,    _lowerCAmelCase=1e-5  ,    _lowerCAmelCase=-1     ):
							# This is a very simplified functional layernorm, designed to duplicate
							# the functionality of PyTorch nn.functional.layer_norm when this is needed to port
							# models in Transformers.

							if weight.shape.rank != 1 or bias.shape.rank != 1 or not isinstance(_lowerCAmelCase  ,    _lowerCAmelCase     ):
														raise NotImplementedError('''Only 1D weight and bias tensors are supported for now, with only a single axis.'''     )

							# Get mean and variance on the axis to be normalized
							SCREAMING_SNAKE_CASE_	,	SCREAMING_SNAKE_CASE_   	=    tf.nn.moments(_lowerCAmelCase  ,    axes=[axis]  ,    keepdims=_lowerCAmelCase     )

							if axis != -1:
														# Reshape scale and weight to have the same rank as inputs, but with 1 dimensions
														# on every dimension except axis
														SCREAMING_SNAKE_CASE_   	=    [1] * inputs.shape.rank
														SCREAMING_SNAKE_CASE_   	=    shape_list(_lowerCAmelCase     )[axis]
														SCREAMING_SNAKE_CASE_   	=    tf.reshape(_lowerCAmelCase  ,    _lowerCAmelCase     )
														SCREAMING_SNAKE_CASE_   	=    tf.reshape(_lowerCAmelCase  ,    _lowerCAmelCase     )

							# Compute layer normalization using the batch_normalization
							# function.
							SCREAMING_SNAKE_CASE_   	=    tf.nn.batch_normalization(
							    _lowerCAmelCase  ,    _lowerCAmelCase  ,    _lowerCAmelCase  ,    offset=_lowerCAmelCase  ,    scale=_lowerCAmelCase  ,    variance_epsilon=_lowerCAmelCase  ,    )
							return outputs

def 			a   (_lowerCAmelCase  ,    _lowerCAmelCase=0  ,    _lowerCAmelCase=-1     ):
							# Replicates the behavior of torch.flatten in TF

							# If end_dim or start_dim is negative, count them from the end
							if end_dim < 0:
														end_dim += input.shape.rank
							if start_dim < 0:
														start_dim += input.shape.rank

							if start_dim == end_dim:
														return input

							SCREAMING_SNAKE_CASE_   	=    tf.shape(_lowerCAmelCase     )
							SCREAMING_SNAKE_CASE_   	=    tf.math.reduce_prod(in_shape[start_dim : end_dim + 1]     )
							SCREAMING_SNAKE_CASE_   	=    tf.concat([in_shape[:start_dim], [flattened_dim], in_shape[end_dim + 1 :]]  ,    axis=0     )
							return tf.reshape(_lowerCAmelCase  ,    _lowerCAmelCase     )

def 			a   (_lowerCAmelCase     ):
							if not isinstance(_lowerCAmelCase  ,    tf.Tensor     ):
														SCREAMING_SNAKE_CASE_   	=    tf.convert_to_tensor(_lowerCAmelCase     )  # Catches stray NumPy inputs
							if encoder_attention_mask.shape.rank == 3:
														SCREAMING_SNAKE_CASE_   	=    encoder_attention_mask[:, None, :, :]
							if encoder_attention_mask.shape.rank == 2:
														SCREAMING_SNAKE_CASE_   	=    encoder_attention_mask[:, None, None, :]
							# T5 has a mask that can compare sequence ids, we can simulate this here with this transposition
							# Cf. https://github.com/tensorflow/mesh/blob/8d2465e9bc93129b913b5ccc6a59aa97abd96ec6/mesh_tensorflow
							# /transformer/transformer_layers.py#L270
							# encoder_extended_attention_mask = (encoder_extended_attention_mask ==
							# encoder_extended_attention_mask.transpose(-1, -2))
							SCREAMING_SNAKE_CASE_   	=    (
							    tf.cast(1  ,    encoder_attention_mask.dtype     ) - encoder_extended_attention_mask
							) * encoder_extended_attention_mask.dtype.min

							return encoder_extended_attention_mask

def 			a   (_lowerCAmelCase  ,    _lowerCAmelCase  ,    _lowerCAmelCase = "input_ids"     ):
							tf.debugging.assert_less(
							    _lowerCAmelCase  ,    tf.cast(_lowerCAmelCase  ,    dtype=tensor.dtype     )  ,    message=(
							        F"The maximum value of {tensor_name} ({tf.math.reduce_max(_lowerCAmelCase     )}) must be smaller than the embedding "
							        F"layer's input dimension ({embed_dim}). The likely cause is some problem at tokenization time."
							    )  ,    )

def 			a   (_lowerCAmelCase  ,    _lowerCAmelCase  ,    _lowerCAmelCase     ):
							SCREAMING_SNAKE_CASE_   	=    6_4_5_1_2
							# Check that no item in `data` is larger than `HDF5_OBJECT_HEADER_LIMIT`
							# because in that case even chunking the array would not make the saving
							# possible.
							SCREAMING_SNAKE_CASE_   	=    [x for x in data if len(_lowerCAmelCase     ) > HDF5_OBJECT_HEADER_LIMIT]

							# Expecting this to never be true.
							if bad_attributes:
														raise RuntimeError(
														    '''The following attributes cannot be saved to HDF5 file because '''
														    F"they are larger than {HDF5_OBJECT_HEADER_LIMIT} "
														    F"bytes: {bad_attributes}"     )

							SCREAMING_SNAKE_CASE_   	=    np.asarray(_lowerCAmelCase     )

							SCREAMING_SNAKE_CASE_   	=    1
							SCREAMING_SNAKE_CASE_   	=    np.array_split(_lowerCAmelCase  ,    _lowerCAmelCase     )

							# This will never loop forever thanks to the test above.
							while any(x.nbytes > HDF5_OBJECT_HEADER_LIMIT for x in chunked_data     ):
														num_chunks += 1
														SCREAMING_SNAKE_CASE_   	=    np.array_split(_lowerCAmelCase  ,    _lowerCAmelCase     )

							if num_chunks > 1:
														for chunk_id, chunk_data in enumerate(_lowerCAmelCase     ):
																					SCREAMING_SNAKE_CASE_   	=    chunk_data
							else:
														SCREAMING_SNAKE_CASE_   	=    data

def 			a   (_lowerCAmelCase  ,    _lowerCAmelCase     ):
							if name in group.attrs:
														SCREAMING_SNAKE_CASE_   	=    [n.decode('''utf8'''     ) if hasattr(_lowerCAmelCase  ,    '''decode'''     ) else n for n in group.attrs[name]]
							else:
														SCREAMING_SNAKE_CASE_   	=    []
														SCREAMING_SNAKE_CASE_   	=    0
														while "%s%d" % (name, chunk_id) in group.attrs:
																					data.extend(
																					    [n.decode('''utf8'''     ) if hasattr(_lowerCAmelCase  ,    '''decode'''     ) else n for n in group.attrs['''%s%d''' % (name, chunk_id)]]     )
																					chunk_id += 1
							return data






def 			a   (_lowerCAmelCase     ):

							def _expand_single_ad_tensor(_lowerCAmelCase     ):
														if isinstance(_lowerCAmelCase  ,    tf.Tensor     ) and t.shape.rank == 1:
																					return tf.expand_dims(_lowerCAmelCase  ,    axis=-1     )
														return t

							return tf.nest.map_structure(_expand_single_ad_tensor  ,    _lowerCAmelCase     )











import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


__SCREAMING_SNAKE_CASE 						=logging.get_logger(__name__)


__SCREAMING_SNAKE_CASE 						="""▁"""

__SCREAMING_SNAKE_CASE 						={"""vocab_file""": """spiece.model"""}

__SCREAMING_SNAKE_CASE 						={
    """vocab_file""": {
        """google/reformer-crime-and-punishment""": (
            """https://huggingface.co/google/reformer-crime-and-punishment/resolve/main/spiece.model"""
        )
    }
}

__SCREAMING_SNAKE_CASE 						={
    """google/reformer-crime-and-punishment""": 524_288,
}






class  __magic_name__						(  __UpperCAmelCase):
				'''simple docstring'''







				SCREAMING_SNAKE_CASE__					:							Any	      =					VOCAB_FILES_NAMES
				SCREAMING_SNAKE_CASE__					:							Union[str, Any]	      =					PRETRAINED_VOCAB_FILES_MAP
				SCREAMING_SNAKE_CASE__					:							Optional[int]	      =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
				SCREAMING_SNAKE_CASE__					:							Union[str, Any]	      =					["input_ids", "attention_mask"]


				def __init__(							self:   List[Any]		,					_lowerCamelCase:   List[Any]		,					_lowerCamelCase:   List[Any]="</s>"		,					_lowerCamelCase:   str="<unk>"		,					_lowerCamelCase:   Union[str, Any]=[]		,					_lowerCamelCase:   Optional[Dict[str, Any]] = None		,					**_lowerCamelCase:   List[str]		,					):
											SCREAMING_SNAKE_CASE_   	=    {} if sp_model_kwargs is None else sp_model_kwargs

											super().__init__(
											    eos_token=_lowerCamelCase		,					unk_token=_lowerCamelCase		,					additional_special_tokens=_lowerCamelCase		,					sp_model_kwargs=self.sp_model_kwargs		,					**_lowerCamelCase		,					)

											SCREAMING_SNAKE_CASE_   	=    vocab_file
											SCREAMING_SNAKE_CASE_   	=    spm.SentencePieceProcessor(**self.sp_model_kwargs		)
											self.sp_model.Load(_lowerCamelCase		)


				@property
				def     _A							(							self:   Optional[int]		):
											return self.sp_model.get_piece_size()


				def     _A							(							self:   int		):
											SCREAMING_SNAKE_CASE_   	=    {self.convert_ids_to_tokens(_lowerCamelCase		): i for i in range(self.vocab_size		)}
											vocab.update(self.added_tokens_encoder		)
											return vocab


				def __getstate__(							self:   Tuple		):
											SCREAMING_SNAKE_CASE_   	=    self.__dict__.copy()
											SCREAMING_SNAKE_CASE_   	=    None
											return state


				def __setstate__(							self:   Dict		,					_lowerCamelCase:   Union[str, Any]		):
											SCREAMING_SNAKE_CASE_   	=    d

											# for backward compatibility
											if not hasattr(self		,					'''sp_model_kwargs'''		):
																		SCREAMING_SNAKE_CASE_   	=    {}

											SCREAMING_SNAKE_CASE_   	=    spm.SentencePieceProcessor(**self.sp_model_kwargs		)
											self.sp_model.Load(self.vocab_file		)


				def     _A							(							self:   Optional[Any]		,					_lowerCamelCase:   str		):
											return self.sp_model.encode(_lowerCamelCase		,					out_type=_lowerCamelCase		)


				def     _A							(							self:   int		,					_lowerCamelCase:   Optional[int]		):
											return self.sp_model.piece_to_id(_lowerCamelCase		)


				def     _A							(							self:   Union[str, Any]		,					_lowerCamelCase:   Tuple		):
											if index < self.sp_model.get_piece_size():
																		SCREAMING_SNAKE_CASE_   	=    self.sp_model.IdToPiece(_lowerCamelCase		)
											return token


				def     _A							(							self:   List[str]		,					_lowerCamelCase:   int		):
											SCREAMING_SNAKE_CASE_   	=    []
											SCREAMING_SNAKE_CASE_   	=    ''''''
											for token in tokens:
																		# make sure that special tokens are not decoded using sentencepiece model
																		if token in self.all_special_tokens:
																									out_string += self.sp_model.decode(_lowerCamelCase		) + token
																									SCREAMING_SNAKE_CASE_   	=    []
																		else:
																									current_sub_tokens.append(_lowerCamelCase		)
											out_string += self.sp_model.decode(_lowerCamelCase		)
											return out_string.strip()





				def     _A							(							self:   Optional[int]		,					_lowerCamelCase:   str		,					_lowerCamelCase:   Optional[str] = None		):
											if not os.path.isdir(_lowerCamelCase		):
																		logger.error(f"Vocabulary path ({save_directory}) should be a directory"		)
																		return
											SCREAMING_SNAKE_CASE_   	=    os.path.join(
											    _lowerCamelCase		,					(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']		)

											if os.path.abspath(self.vocab_file		) != os.path.abspath(_lowerCamelCase		) and os.path.isfile(self.vocab_file		):
																		copyfile(self.vocab_file		,					_lowerCamelCase		)
											elif not os.path.isfile(self.vocab_file		):
																		with open(_lowerCamelCase		,					'''wb'''		) as fi:
																									SCREAMING_SNAKE_CASE_   	=    self.sp_model.serialized_model_proto()
																									fi.write(_lowerCamelCase		)

											return (out_vocab_file,)


