






'''simple docstring'''





import os
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__lowerCamelCase						:    Optional[int]     				=  logging.get_logger(__name__)

__lowerCamelCase						:    Optional[int]     				=  """▁"""

__lowerCamelCase						:    Tuple     				=  {"""vocab_file""": """sentencepiece.bpe.model""", """monolingual_vocab_file""": """dict.txt"""}

__lowerCamelCase						:    Optional[Any]     				=  {
    """vocab_file""": {
        """vinai/bartpho-syllable""": """https://huggingface.co/vinai/bartpho-syllable/resolve/main/sentencepiece.bpe.model""",
    },
    """monolingual_vocab_file""": {
        """vinai/bartpho-syllable""": """https://huggingface.co/vinai/bartpho-syllable/resolve/main/dict.txt""",
    },
}

__lowerCamelCase						:    Optional[int]     				=  {"""vinai/bartpho-syllable""": 1024}



class   lowerCAmelCase__ ( _lowerCAmelCase  ):
			A     				= VOCAB_FILES_NAMES
			A     				= PRETRAINED_VOCAB_FILES_MAP
			A     				= PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
			A     				= ['''input_ids''', '''attention_mask''']

			def __init__(					self					:		Optional[Any]     ,		UpperCamelCase_					:		List[str]     ,		UpperCamelCase_					:		Dict     ,		UpperCamelCase_					:		List[str]="<s>"     ,		UpperCamelCase_					:		List[Any]="</s>"     ,		UpperCamelCase_					:		List[Any]="</s>"     ,		UpperCamelCase_					:		Optional[int]="<s>"     ,		UpperCamelCase_					:		Dict="<unk>"     ,		UpperCamelCase_					:		Optional[int]="<pad>"     ,		UpperCamelCase_					:		Any="<mask>"     ,		UpperCamelCase_					:		Optional[Dict[str, Any]] = None     ,		**UpperCamelCase_					:		Optional[int]     ,		) ->  None:







				"""simple docstring"""






				lowerCamelCase_					:  Optional[Any]						=    AddedToken(UpperCamelCase_     ,		lstrip=UpperCamelCase_     ,		rstrip=UpperCamelCase_	) if isinstance(UpperCamelCase_     ,		UpperCamelCase_	) else mask_token

				lowerCamelCase_					:  Tuple						=    {} if sp_model_kwargs is None else sp_model_kwargs

				super().__init__(
				    bos_token=UpperCamelCase_     ,		eos_token=UpperCamelCase_     ,		unk_token=UpperCamelCase_     ,		sep_token=UpperCamelCase_     ,		cls_token=UpperCamelCase_     ,		pad_token=UpperCamelCase_     ,		mask_token=UpperCamelCase_     ,		sp_model_kwargs=self.sp_model_kwargs     ,		**UpperCamelCase_     ,		)

				lowerCamelCase_					:  str						=    vocab_file
				lowerCamelCase_					:  str						=    monolingual_vocab_file
				lowerCamelCase_					:  str						=    spm.SentencePieceProcessor(**self.sp_model_kwargs	)
				self.sp_model.Load(str(UpperCamelCase_	)	)

				# Load the reduced vocab

				# Keep order of special tokens for backward compatibility
				lowerCamelCase_					:  Optional[int]						=    {}
				lowerCamelCase_					:  Union[str, Any]						=    0
				for token in [bos_token, pad_token, eos_token, unk_token, sep_token, cls_token]:
					if str(UpperCamelCase_	) not in self.fairseq_tokens_to_ids:
						lowerCamelCase_					:  Tuple						=    cnt
						cnt += 1
				with open(UpperCamelCase_     ,		'''r'''     ,		encoding='''utf-8'''	) as f:
					for line in f.readlines():
						lowerCamelCase_					:  Optional[Any]						=    line.strip().split()[0]
						lowerCamelCase_					:  Tuple						=    len(self.fairseq_tokens_to_ids	)
				if str(UpperCamelCase_	) not in self.fairseq_tokens_to_ids:
					lowerCamelCase_					:  Optional[int]						=    len(self.fairseq_tokens_to_ids	)

				lowerCamelCase_					:  Optional[int]						=    {v: k for k, v in self.fairseq_tokens_to_ids.items()}

			def __getstate__(					self					:		List[Any]	) ->  Optional[int]:







				"""simple docstring"""






				lowerCamelCase_					:  Dict						=    self.__dict__.copy()
				lowerCamelCase_					:  List[Any]						=    None
				lowerCamelCase_					:  int						=    self.sp_model.serialized_model_proto()
				return state

			def __setstate__(					self					:		Any     ,		UpperCamelCase_					:		int	) ->  Any:







				"""simple docstring"""






				lowerCamelCase_					:  Tuple						=    d

				# for backward compatibility
				if not hasattr(self     ,		'''sp_model_kwargs'''	):
					lowerCamelCase_					:  Optional[Any]						=    {}

				lowerCamelCase_					:  Union[str, Any]						=    spm.SentencePieceProcessor(**self.sp_model_kwargs	)
				self.sp_model.LoadFromSerializedProto(self.sp_model_proto	)

			def 						__UpperCamelCase				(					self					:		Union[str, Any]     ,		UpperCamelCase_					:		List[int]     ,		UpperCamelCase_					:		Optional[List[int]] = None	) ->  List[int]:







				"""simple docstring"""






				if token_ids_a is None:
					return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
				lowerCamelCase_					:  List[str]						=    [self.cls_token_id]
				lowerCamelCase_					:  int						=    [self.sep_token_id]
				return cls + token_ids_a + sep + sep + token_ids_a + sep

			def 						__UpperCamelCase				(					self					:		Any     ,		UpperCamelCase_					:		List[int]     ,		UpperCamelCase_					:		Optional[List[int]] = None     ,		UpperCamelCase_					:		bool = False	) ->  List[int]:







				"""simple docstring"""






				if already_has_special_tokens:
					return super().get_special_tokens_mask(
					    token_ids_a=UpperCamelCase_     ,		token_ids_a=UpperCamelCase_     ,		already_has_special_tokens=UpperCamelCase_	)

				if token_ids_a is None:
					return [1] + ([0] * len(UpperCamelCase_	)) + [1]
				return [1] + ([0] * len(UpperCamelCase_	)) + [1, 1] + ([0] * len(UpperCamelCase_	)) + [1]

			def 						__UpperCamelCase				(					self					:		Optional[int]     ,		UpperCamelCase_					:		List[int]     ,		UpperCamelCase_					:		Optional[List[int]] = None	) ->  List[int]:







				"""simple docstring"""






				lowerCamelCase_					:  Dict						=    [self.sep_token_id]
				lowerCamelCase_					:  Optional[Any]						=    [self.cls_token_id]

				if token_ids_a is None:
					return len(cls + token_ids_a + sep	) * [0]
				return len(cls + token_ids_a + sep + sep + token_ids_a + sep	) * [0]

			@property
			def 						__UpperCamelCase				(					self					:		Optional[int]	) ->  List[str]:







				"""simple docstring"""






				return len(self.fairseq_ids_to_tokens	)

			def 						__UpperCamelCase				(					self					:		Tuple	) ->  Any:







				"""simple docstring"""






				lowerCamelCase_					:  int						=    {self.convert_ids_to_tokens(UpperCamelCase_	): i for i in range(self.vocab_size	)}
				vocab.update(self.added_tokens_encoder	)
				return vocab

			def 						__UpperCamelCase				(					self					:		Optional[Any]     ,		UpperCamelCase_					:		str	) ->  List[str]:







				"""simple docstring"""






				return self.sp_model.encode(UpperCamelCase_     ,		out_type=UpperCamelCase_	)

			def 						__UpperCamelCase				(					self					:		Dict     ,		UpperCamelCase_					:		Union[str, Any]	) ->  str:







				"""simple docstring"""






				if token in self.fairseq_tokens_to_ids:
					return self.fairseq_tokens_to_ids[token]
				else:
					return self.unk_token_id

			def 						__UpperCamelCase				(					self					:		Optional[int]     ,		UpperCamelCase_					:		int	) ->  Tuple:







				"""simple docstring"""






				return self.fairseq_ids_to_tokens[index]

			def 						__UpperCamelCase				(					self					:		Optional[Any]     ,		UpperCamelCase_					:		Tuple	) ->  Union[str, Any]:







				"""simple docstring"""






				lowerCamelCase_					:  str						=    ''''''.join(UpperCamelCase_	).replace(UpperCamelCase_     ,		''' '''	).strip()
				return out_string





			def 						__UpperCamelCase				(					self					:		List[str]     ,		UpperCamelCase_					:		str     ,		UpperCamelCase_					:		Optional[str] = None	) ->  Tuple[str]:







				"""simple docstring"""






				if not os.path.isdir(UpperCamelCase_	):
					logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""	)
					return
				lowerCamelCase_					:  Union[str, Any]						=    os.path.join(
				    UpperCamelCase_     ,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']	)
				lowerCamelCase_					:  Union[str, Any]						=    os.path.join(
				    UpperCamelCase_     ,		(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''monolingual_vocab_file''']     ,		)

				if os.path.abspath(self.vocab_file	) != os.path.abspath(UpperCamelCase_	) and os.path.isfile(self.vocab_file	):
					copyfile(self.vocab_file     ,		UpperCamelCase_	)
				elif not os.path.isfile(self.vocab_file	):
					with open(UpperCamelCase_     ,		'''wb'''	) as fi:
						lowerCamelCase_					:  List[Any]						=    self.sp_model.serialized_model_proto()
						fi.write(UpperCamelCase_	)

				if os.path.abspath(self.monolingual_vocab_file	) != os.path.abspath(
				    UpperCamelCase_	) and os.path.isfile(self.monolingual_vocab_file	):
					copyfile(self.monolingual_vocab_file     ,		UpperCamelCase_	)
				elif not os.path.isfile(self.monolingual_vocab_file	):
					with open(UpperCamelCase_     ,		'''w'''     ,		encoding='''utf-8'''	) as fp:
						for token in self.fairseq_tokens_to_ids:
							if token not in self.all_special_tokens:
								fp.write(F"""{str(UpperCamelCase_	)} \n"""	)

				return out_vocab_file, out_monolingual_vocab_file












'''simple docstring'''







from torch import nn







def 						_A      ( UpperCAmelCase						):






   '''simple docstring'''







   if act_fn in ["swish", "silu"]:
      return nn.SiLU()
   elif act_fn == "mish":
      return nn.Mish()
   elif act_fn == "gelu":
      return nn.GELU()
   else:
      raise ValueError(F"""Unsupported activation function: {act_fn}"""						)


