


'''simple docstring'''


import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase_						:  Tuple				  =						logging.get_logger(__name__)
UpperCAmelCase_						:  Optional[int]				  =						{"""vocab_file""": """spiece.model"""}

UpperCAmelCase_						:  List[Any]				  =						{
    """vocab_file""": {
        """albert-base-v1""": """https://huggingface.co/albert-base-v1/resolve/main/spiece.model""",
        """albert-large-v1""": """https://huggingface.co/albert-large-v1/resolve/main/spiece.model""",
        """albert-xlarge-v1""": """https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model""",
        """albert-xxlarge-v1""": """https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model""",
        """albert-base-v2""": """https://huggingface.co/albert-base-v2/resolve/main/spiece.model""",
        """albert-large-v2""": """https://huggingface.co/albert-large-v2/resolve/main/spiece.model""",
        """albert-xlarge-v2""": """https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model""",
        """albert-xxlarge-v2""": """https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model""",
    }
}

UpperCAmelCase_						:  Tuple				  =						{
    """albert-base-v1""": 512,
    """albert-large-v1""": 512,
    """albert-xlarge-v1""": 512,
    """albert-xxlarge-v1""": 512,
    """albert-base-v2""": 512,
    """albert-large-v2""": 512,
    """albert-xlarge-v2""": 512,
    """albert-xxlarge-v2""": 512,
}

UpperCAmelCase_						:  Optional[Any]				  =						"""▁"""





class        lowercase__    (		__SCREAMING_SNAKE_CASE   ):





							'''simple docstring'''


							A_				: Dict       =    VOCAB_FILES_NAMES
							A_				: Optional[Any]       =    PRETRAINED_VOCAB_FILES_MAP
							A_				: List[Any]       =    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES






							def __init__(       self       ,   __snake_case       ,   __snake_case=True       ,   __snake_case=True       ,   __snake_case=False       ,   __snake_case="[CLS]"       ,   __snake_case="[SEP]"       ,   __snake_case="<unk>"       ,   __snake_case="[SEP]"       ,   __snake_case="<pad>"       ,   __snake_case="[CLS]"       ,   __snake_case="[MASK]"       ,   __snake_case = None       ,   **__snake_case       ,   ):
											_SCREAMING_SNAKE_CASE					:			Any							= (
											    AddedToken(A_       ,   lstrip=A_       ,   rstrip=A_       ,   normalized=A_    )
											    if isinstance(A_       ,   A_    )
											    else mask_token
											)

											_SCREAMING_SNAKE_CASE					:			Tuple							= {} if sp_model_kwargs is None else sp_model_kwargs

											super().__init__(
											    do_lower_case=A_       ,   remove_space=A_       ,   keep_accents=A_       ,   bos_token=A_       ,   eos_token=A_       ,   unk_token=A_       ,   sep_token=A_       ,   pad_token=A_       ,   cls_token=A_       ,   mask_token=A_       ,   sp_model_kwargs=self.sp_model_kwargs       ,   **A_       ,   )

											_SCREAMING_SNAKE_CASE					:			int							= do_lower_case
											_SCREAMING_SNAKE_CASE					:			Dict							= remove_space
											_SCREAMING_SNAKE_CASE					:			int							= keep_accents
											_SCREAMING_SNAKE_CASE					:			Union[str, Any]							= vocab_file

											_SCREAMING_SNAKE_CASE					:			List[str]							= spm.SentencePieceProcessor(**self.sp_model_kwargs    )
											self.sp_model.Load(A_    )






							@property
							def UpperCAmelCase_      (       self    ):
											return len(self.sp_model    )






							def UpperCAmelCase_      (       self    ):
											_SCREAMING_SNAKE_CASE					:			Any							= {self.convert_ids_to_tokens(A_    ): i for i in range(self.vocab_size    )}
											vocab.update(self.added_tokens_encoder    )
											return vocab






							def __getstate__(       self    ):
											_SCREAMING_SNAKE_CASE					:			Optional[int]							= self.__dict__.copy()
											_SCREAMING_SNAKE_CASE					:			int							= None
											return state






							def __setstate__(       self       ,   __snake_case    ):
											_SCREAMING_SNAKE_CASE					:			List[str]							= d

											# for backward compatibility
											if not hasattr(self       ,   """sp_model_kwargs"""    ):
															_SCREAMING_SNAKE_CASE					:			Union[str, Any]							= {}

											_SCREAMING_SNAKE_CASE					:			Union[str, Any]							= spm.SentencePieceProcessor(**self.sp_model_kwargs    )
											self.sp_model.Load(self.vocab_file    )






							def UpperCAmelCase_      (       self       ,   __snake_case    ):

											if self.remove_space:
															_SCREAMING_SNAKE_CASE					:			List[str]							= """ """.join(inputs.strip().split()    )
											else:
															_SCREAMING_SNAKE_CASE					:			Optional[Any]							= inputs
											_SCREAMING_SNAKE_CASE					:			str							= outputs.replace("""``"""       ,   """\""""    ).replace("""\'\'"""       ,   """\""""    )

											if not self.keep_accents:
															_SCREAMING_SNAKE_CASE					:			Union[str, Any]							= unicodedata.normalize("""NFKD"""       ,   A_    )
															_SCREAMING_SNAKE_CASE					:			List[str]							= """""".join([c for c in outputs if not unicodedata.combining(A_    )]    )
											if self.do_lower_case:
															_SCREAMING_SNAKE_CASE					:			str							= outputs.lower()

											return outputs






							def UpperCAmelCase_      (       self       ,   __snake_case    ):
											_SCREAMING_SNAKE_CASE					:			Union[str, Any]							= self.preprocess_text(A_    )
											_SCREAMING_SNAKE_CASE					:			Any							= self.sp_model.encode(A_       ,   out_type=A_    )
											_SCREAMING_SNAKE_CASE					:			Tuple							= []
											for piece in pieces:
															if len(A_    ) > 1 and piece[-1] == str(""","""    ) and piece[-2].isdigit():
																			_SCREAMING_SNAKE_CASE					:			Optional[int]							= self.sp_model.EncodeAsPieces(piece[:-1].replace(A_       ,   """"""    )    )
																			if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																							if len(cur_pieces[0]    ) == 1:
																											_SCREAMING_SNAKE_CASE					:			Union[str, Any]							= cur_pieces[1:]
																							else:
																											_SCREAMING_SNAKE_CASE					:			str							= cur_pieces[0][1:]
																			cur_pieces.append(piece[-1]    )
																			new_pieces.extend(A_    )
															else:
																			new_pieces.append(A_    )

											return new_pieces






							def UpperCAmelCase_      (       self       ,   __snake_case    ):
											return self.sp_model.PieceToId(A_    )






							def UpperCAmelCase_      (       self       ,   __snake_case    ):
											return self.sp_model.IdToPiece(A_    )






							def UpperCAmelCase_      (       self       ,   __snake_case    ):
											_SCREAMING_SNAKE_CASE					:			Union[str, Any]							= []
											_SCREAMING_SNAKE_CASE					:			Union[str, Any]							= """"""
											_SCREAMING_SNAKE_CASE					:			Optional[int]							= False
											for token in tokens:
															# make sure that special tokens are not decoded using sentencepiece model
															if token in self.all_special_tokens:
																			if not prev_is_special:
																							out_string += " "
																			out_string += self.sp_model.decode(A_    ) + token
																			_SCREAMING_SNAKE_CASE					:			Dict							= True
																			_SCREAMING_SNAKE_CASE					:			List[str]							= []
															else:
																			current_sub_tokens.append(A_    )
																			_SCREAMING_SNAKE_CASE					:			Union[str, Any]							= False
											out_string += self.sp_model.decode(A_    )
											return out_string.strip()






							def UpperCAmelCase_      (       self       ,   __snake_case       ,   __snake_case = None    ):
											_SCREAMING_SNAKE_CASE					:			Optional[Any]							= [self.sep_token_id]
											_SCREAMING_SNAKE_CASE					:			Optional[Any]							= [self.cls_token_id]
											if token_ids_a is None:
															return cls + token_ids_a + sep
											return cls + token_ids_a + sep + token_ids_a + sep






							def UpperCAmelCase_      (       self       ,   __snake_case       ,   __snake_case = None       ,   __snake_case = False    ):

											if already_has_special_tokens:
															return super().get_special_tokens_mask(
															    token_ids_a=A_       ,   token_ids_a=A_       ,   already_has_special_tokens=A_    )

											if token_ids_a is not None:
															return [1] + ([0] * len(A_    )) + [1] + ([0] * len(A_    )) + [1]
											return [1] + ([0] * len(A_    )) + [1]






							def UpperCAmelCase_      (       self       ,   __snake_case       ,   __snake_case = None    ):
											_SCREAMING_SNAKE_CASE					:			Dict							= [self.sep_token_id]
											_SCREAMING_SNAKE_CASE					:			Dict							= [self.cls_token_id]

											if token_ids_a is None:
															return len(cls + token_ids_a + sep    ) * [0]
											return len(cls + token_ids_a + sep    ) * [0] + len(token_ids_a + sep    ) * [1]






							def UpperCAmelCase_      (       self       ,   __snake_case       ,   __snake_case = None    ):

											if not os.path.isdir(A_    ):
															logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""    )
															return
											_SCREAMING_SNAKE_CASE					:			Optional[int]							= os.path.join(
											    A_       ,   (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]    )

											if os.path.abspath(self.vocab_file    ) != os.path.abspath(A_    ) and os.path.isfile(self.vocab_file    ):
															copyfile(self.vocab_file       ,   A_    )
											elif not os.path.isfile(self.vocab_file    ):
															with open(A_       ,   """wb"""    ) as fi:
																			_SCREAMING_SNAKE_CASE					:			List[Any]							= self.sp_model.serialized_model_proto()
																			fi.write(A_    )

											return (out_vocab_file,)











from torch import nn


def 							__snake_case							(    lowerCAmelCase_	)				->						Tuple:
    if act_fn in ["swish", "silu"]:
        return nn.SiLU()
    elif act_fn == "mish":
        return nn.Mish()
    elif act_fn == "gelu":
        return nn.GELU()
    else:
        raise ValueError(f'''Unsupported activation function: {act_fn}'''	)
