

"""simple docstring"""





import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__UpperCamelCase    : Optional[Any]				 =						logging.get_logger(__name__)
__UpperCamelCase    : Optional[int]				 =						{'''vocab_file''': '''spiece.model'''}

__UpperCamelCase    : int				 =						{
    '''vocab_file''': {
        '''albert-base-v1''': '''https://huggingface.co/albert-base-v1/resolve/main/spiece.model''',
        '''albert-large-v1''': '''https://huggingface.co/albert-large-v1/resolve/main/spiece.model''',
        '''albert-xlarge-v1''': '''https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model''',
        '''albert-xxlarge-v1''': '''https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model''',
        '''albert-base-v2''': '''https://huggingface.co/albert-base-v2/resolve/main/spiece.model''',
        '''albert-large-v2''': '''https://huggingface.co/albert-large-v2/resolve/main/spiece.model''',
        '''albert-xlarge-v2''': '''https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model''',
        '''albert-xxlarge-v2''': '''https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model''',
    }
}

__UpperCamelCase    : Optional[int]				 =						{
    '''albert-base-v1''': 5_1_2,
    '''albert-large-v1''': 5_1_2,
    '''albert-xlarge-v1''': 5_1_2,
    '''albert-xxlarge-v1''': 5_1_2,
    '''albert-base-v2''': 5_1_2,
    '''albert-large-v2''': 5_1_2,
    '''albert-xlarge-v2''': 5_1_2,
    '''albert-xxlarge-v2''': 5_1_2,
}

__UpperCamelCase    : Tuple				 =						'''▁'''







class 							SCREAMING_SNAKE_CASE   (						a_			):






					"""simple docstring"""



					lowercase__												=			VOCAB_FILES_NAMES
					lowercase__												=			PRETRAINED_VOCAB_FILES_MAP
					lowercase__												=			PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES







					def __init__(			self     :							int  ,lowercase_     :							Optional[Any]  ,lowercase_     :							List[Any]=True  ,lowercase_     :							Tuple=True  ,lowercase_     :							str=False  ,lowercase_     :							Optional[Any]="[CLS]"  ,lowercase_     :							Any="[SEP]"  ,lowercase_     :							Union[str, Any]="<unk>"  ,lowercase_     :							int="[SEP]"  ,lowercase_     :							List[Any]="<pad>"  ,lowercase_     :							List[Any]="[CLS]"  ,lowercase_     :							int="[MASK]"  ,lowercase_     :							Optional[Dict[str, Any]] = None  ,**lowercase_     :							int  ,):
						# Mask token behave like a normal word, i.e. include the space before it and
						# is included in the raw text, there should be a match in a non-normalized sentence.
						lowerCAmelCase__      :		int    							=      (
						    AddedToken(lowercase_  ,lstrip=lowercase_  ,rstrip=lowercase_  ,normalized=lowercase_						)
						    if isinstance(lowercase_  ,lowercase_						)
						    else mask_token
						)

						lowerCAmelCase__      :		str    							=      {} if sp_model_kwargs is None else sp_model_kwargs

						super().__init__(
						    do_lower_case=lowercase_  ,remove_space=lowercase_  ,keep_accents=lowercase_  ,bos_token=lowercase_  ,eos_token=lowercase_  ,unk_token=lowercase_  ,sep_token=lowercase_  ,pad_token=lowercase_  ,cls_token=lowercase_  ,mask_token=lowercase_  ,sp_model_kwargs=self.sp_model_kwargs  ,**lowercase_  ,)

						lowerCAmelCase__      :		Optional[int]    							=      do_lower_case
						lowerCAmelCase__      :		int    							=      remove_space
						lowerCAmelCase__      :		Tuple    							=      keep_accents
						lowerCAmelCase__      :		List[str]    							=      vocab_file

						lowerCAmelCase__      :		Any    							=      spm.SentencePieceProcessor(**self.sp_model_kwargs						)
						self.sp_model.Load(lowercase_						)







					@property
					def 			__lowerCAmelCase					(			self     :							Union[str, Any]						):
						return len(self.sp_model						)







					def 			__lowerCAmelCase					(			self     :							Union[str, Any]						):
						lowerCAmelCase__      :		List[str]    							=      {self.convert_ids_to_tokens(lowercase_						): i for i in range(self.vocab_size						)}
						vocab.update(self.added_tokens_encoder						)
						return vocab







					def __getstate__(			self     :							List[str]						):
						lowerCAmelCase__      :		int    							=      self.__dict__.copy()
						lowerCAmelCase__      :		List[str]    							=      None
						return state







					def __setstate__(			self     :							List[str]  ,lowercase_     :							List[str]						):
						lowerCAmelCase__      :		Tuple    							=      d

						# for backward compatibility
						if not hasattr(self  ,'''sp_model_kwargs'''						):
							lowerCAmelCase__      :		str    							=      {}

						lowerCAmelCase__      :		List[str]    							=      spm.SentencePieceProcessor(**self.sp_model_kwargs						)
						self.sp_model.Load(self.vocab_file						)







					def 			__lowerCAmelCase					(			self     :							Any  ,lowercase_     :							List[str]						):
						if self.remove_space:
							lowerCAmelCase__      :		List[str]    							=      ''' '''.join(inputs.strip().split()						)
						else:
							lowerCAmelCase__      :		Union[str, Any]    							=      inputs
						lowerCAmelCase__      :		Optional[Any]    							=      outputs.replace('''``'''  ,'''"'''						).replace('''\'\''''  ,'''"'''						)

						if not self.keep_accents:
							lowerCAmelCase__      :		Tuple    							=      unicodedata.normalize('''NFKD'''  ,lowercase_						)
							lowerCAmelCase__      :		str    							=      ''''''.join([c for c in outputs if not unicodedata.combining(lowercase_						)]						)
						if self.do_lower_case:
							lowerCAmelCase__      :		Optional[Any]    							=      outputs.lower()

						return outputs







					def 			__lowerCAmelCase					(			self     :							Union[str, Any]  ,lowercase_     :							str						):
						lowerCAmelCase__      :		Optional[Any]    							=      self.preprocess_text(lowercase_						)
						lowerCAmelCase__      :		Union[str, Any]    							=      self.sp_model.encode(lowercase_  ,out_type=lowercase_						)
						lowerCAmelCase__      :		int    							=      []
						for piece in pieces:
							if len(lowercase_						) > 1 and piece[-1] == str(''','''						) and piece[-2].isdigit():
								lowerCAmelCase__      :		Optional[Any]    							=      self.sp_model.EncodeAsPieces(piece[:-1].replace(lowercase_  ,''''''						)						)
								if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
									if len(cur_pieces[0]						) == 1:
										lowerCAmelCase__      :		Optional[Any]    							=      cur_pieces[1:]
									else:
										lowerCAmelCase__      :		Any    							=      cur_pieces[0][1:]
								cur_pieces.append(piece[-1]						)
								new_pieces.extend(lowercase_						)
							else:
								new_pieces.append(lowercase_						)

						return new_pieces







					def 			__lowerCAmelCase					(			self     :							List[str]  ,lowercase_     :							Tuple						):
						return self.sp_model.PieceToId(lowercase_						)







					def 			__lowerCAmelCase					(			self     :							Optional[int]  ,lowercase_     :							Union[str, Any]						):
						return self.sp_model.IdToPiece(lowercase_						)







					def 			__lowerCAmelCase					(			self     :							Optional[Any]  ,lowercase_     :							Tuple						):
						lowerCAmelCase__      :		int    							=      []
						lowerCAmelCase__      :		int    							=      ''''''
						lowerCAmelCase__      :		List[str]    							=      False
						for token in tokens:
							# make sure that special tokens are not decoded using sentencepiece model
							if token in self.all_special_tokens:
								if not prev_is_special:
									out_string += " "
								out_string += self.sp_model.decode(lowercase_						) + token
								lowerCAmelCase__      :		Optional[int]    							=      True
								lowerCAmelCase__      :		List[str]    							=      []
							else:
								current_sub_tokens.append(lowercase_						)
								lowerCAmelCase__      :		Optional[int]    							=      False
						out_string += self.sp_model.decode(lowercase_						)
						return out_string.strip()







					def 			__lowerCAmelCase					(			self     :							Tuple  ,lowercase_     :							List[int]  ,lowercase_     :							Optional[List[int]] = None						):
						lowerCAmelCase__      :		Optional[Any]    							=      [self.sep_token_id]
						lowerCAmelCase__      :		Union[str, Any]    							=      [self.cls_token_id]
						if token_ids_a is None:
							return cls + token_ids_a + sep
						return cls + token_ids_a + sep + token_ids_a + sep







					def 			__lowerCAmelCase					(			self     :							List[str]  ,lowercase_     :							List[int]  ,lowercase_     :							Optional[List[int]] = None  ,lowercase_     :							bool = False						):

						if already_has_special_tokens:
							return super().get_special_tokens_mask(
							    token_ids_a=lowercase_  ,token_ids_a=lowercase_  ,already_has_special_tokens=lowercase_						)

						if token_ids_a is not None:
							return [1] + ([0] * len(lowercase_						)) + [1] + ([0] * len(lowercase_						)) + [1]
						return [1] + ([0] * len(lowercase_						)) + [1]







					def 			__lowerCAmelCase					(			self     :							int  ,lowercase_     :							List[int]  ,lowercase_     :							Optional[List[int]] = None						):
						lowerCAmelCase__      :		Optional[int]    							=      [self.sep_token_id]
						lowerCAmelCase__      :		List[Any]    							=      [self.cls_token_id]

						if token_ids_a is None:
							return len(cls + token_ids_a + sep						) * [0]
						return len(cls + token_ids_a + sep						) * [0] + len(token_ids_a + sep						) * [1]







					def 			__lowerCAmelCase					(			self     :							str  ,lowercase_     :							str  ,lowercase_     :							Optional[str] = None						):
						if not os.path.isdir(lowercase_						):
							logger.error(F'Vocabulary path ({save_directory}) should be a directory'						)
							return
						lowerCAmelCase__      :		Union[str, Any]    							=      os.path.join(
						    lowercase_  ,(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']						)

						if os.path.abspath(self.vocab_file						) != os.path.abspath(lowercase_						) and os.path.isfile(self.vocab_file						):
							copyfile(self.vocab_file  ,lowercase_						)
						elif not os.path.isfile(self.vocab_file						):
							with open(lowercase_  ,'''wb'''						) as fi:
								lowerCAmelCase__      :		List[str]    							=      self.sp_model.serialized_model_proto()
								fi.write(lowercase_						)

						return (out_vocab_file,)







"""simple docstring"""





from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_sentencepiece_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


__UpperCamelCase    : List[str]				 =						{'''configuration_xglm''': ['''XGLM_PRETRAINED_CONFIG_ARCHIVE_MAP''', '''XGLMConfig''']}

try:
							if not is_sentencepiece_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__UpperCamelCase    : Union[str, Any]				 =						['''XGLMTokenizer''']

try:
							if not is_tokenizers_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__UpperCamelCase    : str				 =						['''XGLMTokenizerFast''']

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__UpperCamelCase    : Optional[int]				 =						[
							    '''XGLM_PRETRAINED_MODEL_ARCHIVE_LIST''',
							    '''XGLMForCausalLM''',
							    '''XGLMModel''',
							    '''XGLMPreTrainedModel''',
							]


try:
							if not is_flax_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__UpperCamelCase    : str				 =						[
							    '''FlaxXGLMForCausalLM''',
							    '''FlaxXGLMModel''',
							    '''FlaxXGLMPreTrainedModel''',
							]


try:
							if not is_tf_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							__UpperCamelCase    : str				 =						[
							    '''TF_XGLM_PRETRAINED_MODEL_ARCHIVE_LIST''',
							    '''TFXGLMForCausalLM''',
							    '''TFXGLMModel''',
							    '''TFXGLMPreTrainedModel''',
							]


if TYPE_CHECKING:
							from .configuration_xglm import XGLM_PRETRAINED_CONFIG_ARCHIVE_MAP, XGLMConfig

							try:
														if not is_sentencepiece_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .tokenization_xglm import XGLMTokenizer

							try:
														if not is_tokenizers_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .tokenization_xglm_fast import XGLMTokenizerFast

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_xglm import XGLM_PRETRAINED_MODEL_ARCHIVE_LIST, XGLMForCausalLM, XGLMModel, XGLMPreTrainedModel

							try:
														if not is_flax_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_flax_xglm import FlaxXGLMForCausalLM, FlaxXGLMModel, FlaxXGLMPreTrainedModel

							try:
														if not is_tf_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_tf_xglm import (
														    TF_XGLM_PRETRAINED_MODEL_ARCHIVE_LIST,
														    TFXGLMForCausalLM,
														    TFXGLMModel,
														    TFXGLMPreTrainedModel,
														)


else:
							import sys

							__UpperCamelCase    : Optional[int]				 =						_LazyModule(__name__, globals()['''__file__'''], _import_structure)




