



'''simple docstring'''

import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


a							   =				logging.get_logger(__name__)

a							   =				'▁'

a							   =				{'vocab_file': 'sentencepiece.bpe.model', 'monolingual_vocab_file': 'dict.txt'}

a							   =				{
    'vocab_file': {
        'vinai/bartpho-syllable': 'https://huggingface.co/vinai/bartpho-syllable/resolve/main/sentencepiece.bpe.model',
    },
    'monolingual_vocab_file': {
        'vinai/bartpho-syllable': 'https://huggingface.co/vinai/bartpho-syllable/resolve/main/dict.txt',
    },
}

a							   =				{'vinai/bartpho-syllable': 1_024}






class 						a_		(     snake_case    ):
							UpperCAmelCase			:		Dict        =				VOCAB_FILES_NAMES
							UpperCAmelCase			:		str        =				PRETRAINED_VOCAB_FILES_MAP
							UpperCAmelCase			:		List[Any]        =				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							UpperCAmelCase			:		List[Any]        =				["""input_ids""", """attention_mask"""]


							def __init__(				self   :						Any      ,						a_   :						Optional[int]      ,						a_   :						Optional[int]      ,						a_   :						Optional[Any]="<s>"      ,						a_   :						Any="</s>"      ,						a_   :						Any="</s>"      ,						a_   :						Any="<s>"      ,						a_   :						Union[str, Any]="<unk>"      ,						a_   :						Optional[int]="<pad>"      ,						a_   :						Optional[Any]="<mask>"      ,						a_   :						Optional[Dict[str, Any]] = None      ,						**a_   :						str      ,						)			->					None:
										# Mask token behave like a normal word, i.e. include the space before it
										snake_case:    Tuple      							=AddedToken(a_      ,						lstrip=a_      ,						rstrip=a_			) if isinstance(a_      ,						a_			) else mask_token

										snake_case:    Any      							={} if sp_model_kwargs is None else sp_model_kwargs

										super().__init__(
										    bos_token=a_      ,						eos_token=a_      ,						unk_token=a_      ,						sep_token=a_      ,						cls_token=a_      ,						pad_token=a_      ,						mask_token=a_      ,						sp_model_kwargs=self.sp_model_kwargs      ,						**a_      ,						)

										snake_case:    Tuple      							=vocab_file
										snake_case:    Any      							=monolingual_vocab_file
										snake_case:    Optional[Any]      							=spm.SentencePieceProcessor(**self.sp_model_kwargs			)
										self.sp_model.Load(str(a_			)			)

										# Load the reduced vocab

										# Keep order of special tokens for backward compatibility
										snake_case:    Dict      							={}
										snake_case:    int      							=0
										for token in [bos_token, pad_token, eos_token, unk_token, sep_token, cls_token]:
													if str(a_			) not in self.fairseq_tokens_to_ids:
																snake_case:    Tuple      							=cnt
																cnt += 1
										with open(a_      ,						'r'      ,						encoding='utf-8'			) as f:
													for line in f.readlines():
																snake_case:    int      							=line.strip().split()[0]
																snake_case:    str      							=len(self.fairseq_tokens_to_ids			)
										if str(a_			) not in self.fairseq_tokens_to_ids:
													snake_case:    List[str]      							=len(self.fairseq_tokens_to_ids			)

										snake_case:    Any      							={v: k for k, v in self.fairseq_tokens_to_ids.items()}


							def __getstate__(				self   :						Any			)			->					List[str]:
										snake_case:    List[Any]      							=self.__dict__.copy()
										snake_case:    Any      							=None
										snake_case:    Any      							=self.sp_model.serialized_model_proto()
										return state


							def __setstate__(				self   :						Optional[Any]      ,						a_   :						Optional[int]			)			->					int:
										snake_case:    int      							=d

										# for backward compatibility
										if not hasattr(self      ,						'sp_model_kwargs'			):
													snake_case:    List[str]      							={}

										snake_case:    Optional[int]      							=spm.SentencePieceProcessor(**self.sp_model_kwargs			)
										self.sp_model.LoadFromSerializedProto(self.sp_model_proto			)


							def    UpperCamelCase     (				self   :						List[str]      ,						a_   :						List[int]      ,						a_   :						Optional[List[int]] = None			)			->					List[int]:

										if token_ids_a is None:
													return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
										snake_case:    str      							=[self.cls_token_id]
										snake_case:    List[str]      							=[self.sep_token_id]
										return cls + token_ids_a + sep + sep + token_ids_a + sep


							def    UpperCamelCase     (				self   :						str      ,						a_   :						List[int]      ,						a_   :						Optional[List[int]] = None      ,						a_   :						bool = False			)			->					List[int]:

										if already_has_special_tokens:
													return super().get_special_tokens_mask(
													    token_ids_a=a_      ,						token_ids_a=a_      ,						already_has_special_tokens=a_			)

										if token_ids_a is None:
													return [1] + ([0] * len(a_			)) + [1]
										return [1] + ([0] * len(a_			)) + [1, 1] + ([0] * len(a_			)) + [1]


							def    UpperCamelCase     (				self   :						Union[str, Any]      ,						a_   :						List[int]      ,						a_   :						Optional[List[int]] = None			)			->					List[int]:
										snake_case:    int      							=[self.sep_token_id]
										snake_case:    Tuple      							=[self.cls_token_id]

										if token_ids_a is None:
													return len(cls + token_ids_a + sep			) * [0]
										return len(cls + token_ids_a + sep + sep + token_ids_a + sep			) * [0]


							@property
							def    UpperCamelCase     (				self   :						List[str]			)			->					List[str]:
										return len(self.fairseq_ids_to_tokens			)


							def    UpperCamelCase     (				self   :						Optional[Any]			)			->					Optional[int]:
										snake_case:    int      							={self.convert_ids_to_tokens(a_			): i for i in range(self.vocab_size			)}
										vocab.update(self.added_tokens_encoder			)
										return vocab


							def    UpperCamelCase     (				self   :						List[Any]      ,						a_   :						str			)			->					List[str]:
										return self.sp_model.encode(a_      ,						out_type=a_			)


							def    UpperCamelCase     (				self   :						Dict      ,						a_   :						str			)			->					Dict:
										if token in self.fairseq_tokens_to_ids:
													return self.fairseq_tokens_to_ids[token]
										else:
													return self.unk_token_id


							def    UpperCamelCase     (				self   :						str      ,						a_   :						Dict			)			->					Any:
										return self.fairseq_ids_to_tokens[index]


							def    UpperCamelCase     (				self   :						Union[str, Any]      ,						a_   :						Optional[int]			)			->					Optional[int]:
										snake_case:    Tuple      							=''.join(a_			).replace(a_      ,						' '			).strip()
										return out_string




							def    UpperCamelCase     (				self   :						str      ,						a_   :						str      ,						a_   :						Optional[str] = None			)			->					Tuple[str]:
										if not os.path.isdir(a_			):
													logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''			)
													return
										snake_case:    Optional[int]      							=os.path.join(
										    a_      ,						(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']			)
										snake_case:    List[Any]      							=os.path.join(
										    a_      ,						(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['monolingual_vocab_file']      ,						)

										if os.path.abspath(self.vocab_file			) != os.path.abspath(a_			) and os.path.isfile(self.vocab_file			):
													copyfile(self.vocab_file      ,						a_			)
										elif not os.path.isfile(self.vocab_file			):
													with open(a_      ,						'wb'			) as fi:
																snake_case:    Dict      							=self.sp_model.serialized_model_proto()
																fi.write(a_			)

										if os.path.abspath(self.monolingual_vocab_file			) != os.path.abspath(
										    a_			) and os.path.isfile(self.monolingual_vocab_file			):
													copyfile(self.monolingual_vocab_file      ,						a_			)
										elif not os.path.isfile(self.monolingual_vocab_file			):
													with open(a_      ,						'w'      ,						encoding='utf-8'			) as fp:
																for token in self.fairseq_tokens_to_ids:
																			if token not in self.all_special_tokens:
																						fp.write(F'''{str(a_			)} \n'''			)

										return out_vocab_file, out_monolingual_vocab_file









'''simple docstring'''

import math

import tensorflow as tf
from packaging import version







def a_     (    __UpperCAmelCase					)				->					Optional[Any]:





			"""simple docstring"""







			snake_case:    str      							=tf.convert_to_tensor(__UpperCAmelCase					)
			snake_case:    int      							=0.5 * (1.0 + tf.math.erf(x / tf.cast(tf.sqrt(2.0					)				,							x.dtype					)					))

			return x * cdf







def a_     (    __UpperCAmelCase					)				->					Dict:





			"""simple docstring"""







			snake_case:    Optional[int]      							=tf.convert_to_tensor(__UpperCAmelCase					)
			snake_case:    Tuple      							=tf.cast(math.pi				,							x.dtype					)
			snake_case:    Dict      							=tf.cast(0.04_4715				,							x.dtype					)
			snake_case:    List[str]      							=0.5 * (1.0 + tf.tanh(tf.sqrt(2.0 / pi					) * (x + coeff * tf.pow(__UpperCAmelCase				,							3					))					))

			return x * cdf







def a_     (    __UpperCAmelCase					)				->					Tuple:





			"""simple docstring"""







			snake_case:    Optional[int]      							=tf.convert_to_tensor(__UpperCAmelCase					)

			return x * tf.tanh(tf.math.softplus(__UpperCAmelCase					)					)







def a_     (    __UpperCAmelCase					)				->					Dict:





			"""simple docstring"""







			snake_case:    List[str]      							=tf.convert_to_tensor(__UpperCAmelCase					)
			snake_case:    Dict      							=tf.cast(0.04_4715				,							x.dtype					)
			snake_case:    List[str]      							=tf.cast(0.79_7884_5608				,							x.dtype					)

			return 0.5 * x * (1.0 + tf.tanh(x * coeffa * (1.0 + coeffa * x * x)					))







def a_     (    __UpperCAmelCase					)				->					Optional[Any]:





			"""simple docstring"""







			snake_case:    Union[str, Any]      							=tf.convert_to_tensor(__UpperCAmelCase					)
			snake_case:    Tuple      							=tf.cast(1.702				,							x.dtype					)
			return x * tf.math.sigmoid(coeff * x					)







def a_     (    __UpperCAmelCase					)				->					int:





			"""simple docstring"""







			return tf.clip_by_value(_gelu(__UpperCAmelCase					)				,							-10				,							10					)







def a_     (    __UpperCAmelCase				,							__UpperCAmelCase=-1					)				->					List[str]:





			"""simple docstring"""







			snake_case							,  snake_case:    int      							=tf.split(__UpperCAmelCase				,							2				,							axis=__UpperCAmelCase					)
			return a * tf.math.sigmoid(__UpperCAmelCase					)


if version.parse(tf.version.VERSION) >= version.parse('2.4'):







	def a_     (    __UpperCAmelCase					)				->					int:





				"""simple docstring"""







				return tf.keras.activations.gelu(__UpperCAmelCase				,							approximate=__UpperCAmelCase					)

	a							   =				tf.keras.activations.gelu







	a							   =				approximate_gelu_wrap
else:
	a							   =				_gelu
	a							   =				_gelu_new


a							   =				{
    'gelu': gelu,
    'gelu_10': gelu_aa,
    'gelu_fast': gelu_fast,
    'gelu_new': gelu_new,
    'glu': glu,
    'mish': mish,
    'quick_gelu': quick_gelu,
    'relu': tf.keras.activations.relu,
    'sigmoid': tf.keras.activations.sigmoid,
    'silu': tf.keras.activations.swish,
    'swish': tf.keras.activations.swish,
    'tanh': tf.keras.activations.tanh,
}







def a_     (    __UpperCAmelCase					)				->					Dict:





			"""simple docstring"""







			if activation_string in ACTaFN:
						return ACTaFN[activation_string]
			else:
						raise KeyError(f'''function {activation_string} not found in ACT2FN mapping {list(ACTaFN.keys()					)}'''					)




