






import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowercase  :							List[str]			  =    logging.get_logger(__name__)
lowercase  :							int			  =    {'''vocab_file''': '''spiece.model'''}

lowercase  :							Tuple			  =    {
    '''vocab_file''': {
        '''albert-base-v1''': '''https://huggingface.co/albert-base-v1/resolve/main/spiece.model''',
        '''albert-large-v1''': '''https://huggingface.co/albert-large-v1/resolve/main/spiece.model''',
        '''albert-xlarge-v1''': '''https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model''',
        '''albert-xxlarge-v1''': '''https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model''',
        '''albert-base-v2''': '''https://huggingface.co/albert-base-v2/resolve/main/spiece.model''',
        '''albert-large-v2''': '''https://huggingface.co/albert-large-v2/resolve/main/spiece.model''',
        '''albert-xlarge-v2''': '''https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model''',
        '''albert-xxlarge-v2''': '''https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model''',
    }
}

lowercase  :							List[str]			  =    {
    '''albert-base-v1''': 5_12,
    '''albert-large-v1''': 5_12,
    '''albert-xlarge-v1''': 5_12,
    '''albert-xxlarge-v1''': 5_12,
    '''albert-base-v2''': 5_12,
    '''albert-large-v2''': 5_12,
    '''albert-xlarge-v2''': 5_12,
    '''albert-xxlarge-v2''': 5_12,
}

lowercase  :							Any			  =    '''▁'''




class 		UpperCAmelCase_						(	SCREAMING_SNAKE_CASE__					):


						'''simple docstring'''




						A     :   Tuple						=					VOCAB_FILES_NAMES
						A     :   Union[str, Any]						=					PRETRAINED_VOCAB_FILES_MAP
						A     :   List[Any]						=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES






						def __init__(	self		,				_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE=True		,				_SCREAMING_SNAKE_CASE=True		,				_SCREAMING_SNAKE_CASE=False		,				_SCREAMING_SNAKE_CASE="[CLS]"		,				_SCREAMING_SNAKE_CASE="[SEP]"		,				_SCREAMING_SNAKE_CASE="<unk>"		,				_SCREAMING_SNAKE_CASE="[SEP]"		,				_SCREAMING_SNAKE_CASE="<pad>"		,				_SCREAMING_SNAKE_CASE="[CLS]"		,				_SCREAMING_SNAKE_CASE="[MASK]"		,				_SCREAMING_SNAKE_CASE = None		,				**_SCREAMING_SNAKE_CASE		,				) ->				None:
								# Mask token behave like a normal word, i.e. include the space before it and
								# is included in the raw text, there should be a match in a non-normalized sentence.
								snake_case_				:							Optional[int]						 =       (
								    AddedToken(_SCREAMING_SNAKE_CASE		,				lstrip=_SCREAMING_SNAKE_CASE		,				rstrip=_SCREAMING_SNAKE_CASE		,				normalized=_SCREAMING_SNAKE_CASE				)
								    if isinstance(_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE				)
								    else mask_token
								)

								snake_case_				:							Optional[int]						 =       {} if sp_model_kwargs is None else sp_model_kwargs

								super().__init__(
								    do_lower_case=_SCREAMING_SNAKE_CASE		,				remove_space=_SCREAMING_SNAKE_CASE		,				keep_accents=_SCREAMING_SNAKE_CASE		,				bos_token=_SCREAMING_SNAKE_CASE		,				eos_token=_SCREAMING_SNAKE_CASE		,				unk_token=_SCREAMING_SNAKE_CASE		,				sep_token=_SCREAMING_SNAKE_CASE		,				pad_token=_SCREAMING_SNAKE_CASE		,				cls_token=_SCREAMING_SNAKE_CASE		,				mask_token=_SCREAMING_SNAKE_CASE		,				sp_model_kwargs=self.sp_model_kwargs		,				**_SCREAMING_SNAKE_CASE		,				)

								snake_case_				:							List[Any]						 =       do_lower_case
								snake_case_				:							List[str]						 =       remove_space
								snake_case_				:							List[Any]						 =       keep_accents
								snake_case_				:							List[Any]						 =       vocab_file

								snake_case_				:							str						 =       spm.SentencePieceProcessor(**self.sp_model_kwargs				)
								self.sp_model.Load(_SCREAMING_SNAKE_CASE				)






						@property
						def       _lowerCAmelCase       (	self				) ->				Union[str, Any]:
								return len(self.sp_model				)






						def       _lowerCAmelCase       (	self				) ->				str:
								snake_case_				:							str						 =       {self.convert_ids_to_tokens(_SCREAMING_SNAKE_CASE				): i for i in range(self.vocab_size				)}
								vocab.update(self.added_tokens_encoder				)
								return vocab






						def __getstate__(	self				) ->				Optional[int]:
								snake_case_				:							Optional[Any]						 =       self.__dict__.copy()
								snake_case_				:							List[str]						 =       None
								return state






						def __setstate__(	self		,				_SCREAMING_SNAKE_CASE				) ->				Union[str, Any]:
								snake_case_				:							Union[str, Any]						 =       d

								# for backward compatibility
								if not hasattr(self		,				"sp_model_kwargs"				):
										snake_case_				:							str						 =       {}

								snake_case_				:							str						 =       spm.SentencePieceProcessor(**self.sp_model_kwargs				)
								self.sp_model.Load(self.vocab_file				)






						def       _lowerCAmelCase       (	self		,				_SCREAMING_SNAKE_CASE				) ->				int:
								if self.remove_space:
										snake_case_				:							Optional[int]						 =       " ".join(inputs.strip().split()				)
								else:
										snake_case_				:							Tuple						 =       inputs
								snake_case_				:							Tuple						 =       outputs.replace("``"		,				"\""				).replace("''"		,				"\""				)

								if not self.keep_accents:
										snake_case_				:							Any						 =       unicodedata.normalize("NFKD"		,				_SCREAMING_SNAKE_CASE				)
										snake_case_				:							Optional[int]						 =       "".join([c for c in outputs if not unicodedata.combining(_SCREAMING_SNAKE_CASE				)]				)
								if self.do_lower_case:
										snake_case_				:							Optional[int]						 =       outputs.lower()

								return outputs






						def       _lowerCAmelCase       (	self		,				_SCREAMING_SNAKE_CASE				) ->				List[str]:
								snake_case_				:							Union[str, Any]						 =       self.preprocess_text(_SCREAMING_SNAKE_CASE				)
								snake_case_				:							Optional[Any]						 =       self.sp_model.encode(_SCREAMING_SNAKE_CASE		,				out_type=_SCREAMING_SNAKE_CASE				)
								snake_case_				:							Union[str, Any]						 =       []
								for piece in pieces:
										if len(_SCREAMING_SNAKE_CASE				) > 1 and piece[-1] == str(","				) and piece[-2].isdigit():
												snake_case_				:							str						 =       self.sp_model.EncodeAsPieces(piece[:-1].replace(_SCREAMING_SNAKE_CASE		,				""				)				)
												if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
														if len(cur_pieces[0]				) == 1:
																snake_case_				:							Dict						 =       cur_pieces[1:]
														else:
																snake_case_				:							List[Any]						 =       cur_pieces[0][1:]
												cur_pieces.append(piece[-1]				)
												new_pieces.extend(_SCREAMING_SNAKE_CASE				)
										else:
												new_pieces.append(_SCREAMING_SNAKE_CASE				)

								return new_pieces






						def       _lowerCAmelCase       (	self		,				_SCREAMING_SNAKE_CASE				) ->				Any:
								return self.sp_model.PieceToId(_SCREAMING_SNAKE_CASE				)






						def       _lowerCAmelCase       (	self		,				_SCREAMING_SNAKE_CASE				) ->				int:
								return self.sp_model.IdToPiece(_SCREAMING_SNAKE_CASE				)






						def       _lowerCAmelCase       (	self		,				_SCREAMING_SNAKE_CASE				) ->				Union[str, Any]:
								snake_case_				:							List[Any]						 =       []
								snake_case_				:							str						 =       ""
								snake_case_				:							Any						 =       False
								for token in tokens:
										# make sure that special tokens are not decoded using sentencepiece model
										if token in self.all_special_tokens:
												if not prev_is_special:
														out_string += " "
												out_string += self.sp_model.decode(_SCREAMING_SNAKE_CASE				) + token
												snake_case_				:							Dict						 =       True
												snake_case_				:							Optional[int]						 =       []
										else:
												current_sub_tokens.append(_SCREAMING_SNAKE_CASE				)
												snake_case_				:							Dict						 =       False
								out_string += self.sp_model.decode(_SCREAMING_SNAKE_CASE				)
								return out_string.strip()






						def       _lowerCAmelCase       (	self		,				_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE = None				) ->				List[int]:
								snake_case_				:							int						 =       [self.sep_token_id]
								snake_case_				:							str						 =       [self.cls_token_id]
								if token_ids_a is None:
										return cls + token_ids_a + sep
								return cls + token_ids_a + sep + token_ids_a + sep






						def       _lowerCAmelCase       (	self		,				_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE = None		,				_SCREAMING_SNAKE_CASE = False				) ->				List[int]:

								if already_has_special_tokens:
										return super().get_special_tokens_mask(
										    token_ids_a=_SCREAMING_SNAKE_CASE		,				token_ids_a=_SCREAMING_SNAKE_CASE		,				already_has_special_tokens=_SCREAMING_SNAKE_CASE				)

								if token_ids_a is not None:
										return [1] + ([0] * len(_SCREAMING_SNAKE_CASE				)) + [1] + ([0] * len(_SCREAMING_SNAKE_CASE				)) + [1]
								return [1] + ([0] * len(_SCREAMING_SNAKE_CASE				)) + [1]






						def       _lowerCAmelCase       (	self		,				_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE = None				) ->				List[int]:
								snake_case_				:							Optional[Any]						 =       [self.sep_token_id]
								snake_case_				:							Tuple						 =       [self.cls_token_id]

								if token_ids_a is None:
										return len(cls + token_ids_a + sep				) * [0]
								return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]






						def       _lowerCAmelCase       (	self		,				_SCREAMING_SNAKE_CASE		,				_SCREAMING_SNAKE_CASE = None				) ->				Tuple[str]:
								if not os.path.isdir(_SCREAMING_SNAKE_CASE				):
										logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''				)
										return
								snake_case_				:							Tuple						 =       os.path.join(
								    _SCREAMING_SNAKE_CASE		,				(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]				)

								if os.path.abspath(self.vocab_file				) != os.path.abspath(_SCREAMING_SNAKE_CASE				) and os.path.isfile(self.vocab_file				):
										copyfile(self.vocab_file		,				_SCREAMING_SNAKE_CASE				)
								elif not os.path.isfile(self.vocab_file				):
										with open(_SCREAMING_SNAKE_CASE		,				"wb"				) as fi:
												snake_case_				:							List[Any]						 =       self.sp_model.serialized_model_proto()
												fi.write(_SCREAMING_SNAKE_CASE				)

								return (out_vocab_file,)








from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


lowercase  :							Optional[int]			  =    {
    '''configuration_nllb_moe''': [
        '''NLLB_MOE_PRETRAINED_CONFIG_ARCHIVE_MAP''',
        '''NllbMoeConfig''',
    ]
}

try:
							if not is_torch_available():
														raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
							pass
else:
							lowercase  :							Optional[Any]			  =    [
							    '''NLLB_MOE_PRETRAINED_MODEL_ARCHIVE_LIST''',
							    '''NllbMoeForConditionalGeneration''',
							    '''NllbMoeModel''',
							    '''NllbMoePreTrainedModel''',
							    '''NllbMoeTop2Router''',
							    '''NllbMoeSparseMLP''',
							]


if TYPE_CHECKING:
							from .configuration_nllb_moe import (
							    NLLB_MOE_PRETRAINED_CONFIG_ARCHIVE_MAP,
							    NllbMoeConfig,
							)

							try:
														if not is_torch_available():
																					raise OptionalDependencyNotAvailable()
							except OptionalDependencyNotAvailable:
														pass
							else:
														from .modeling_nllb_moe import (
														    NLLB_MOE_PRETRAINED_MODEL_ARCHIVE_LIST,
														    NllbMoeForConditionalGeneration,
														    NllbMoeModel,
														    NllbMoePreTrainedModel,
														    NllbMoeSparseMLP,
														    NllbMoeTopaRouter,
														)


else:
							import sys

							lowercase  :							Tuple			  =    _LazyModule(__name__, globals()['''__file__'''], _import_structure, module_spec=__spec__)
