





import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import SPIECE_UNDERLINE, logging


UpperCamelCase__											= logging.get_logger(__name__)

UpperCamelCase__											= {"vocab_file": "spiece.model"}

UpperCamelCase__											= {
    "vocab_file": {
        "xlnet-base-cased": "https://huggingface.co/xlnet-base-cased/resolve/main/spiece.model",
        "xlnet-large-cased": "https://huggingface.co/xlnet-large-cased/resolve/main/spiece.model",
    }
}

UpperCamelCase__											= {
    "xlnet-base-cased": None,
    "xlnet-large-cased": None,
}

# Segments (not really needed)
UpperCamelCase__											= 0
UpperCamelCase__											= 1
UpperCamelCase__											= 2
UpperCamelCase__											= 3
UpperCamelCase__											= 4

class        __SCREAMING_SNAKE_CASE					(     _a       ):
							snake_case     :      Tuple						     =       VOCAB_FILES_NAMES
							snake_case     :      Any						     =       PRETRAINED_VOCAB_FILES_MAP
							snake_case     :      Tuple						     =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							snake_case     :      Tuple						     =       """left"""






							def __init__(						self  , __lowerCAmelCase  , __lowerCAmelCase=False  , __lowerCAmelCase=True  , __lowerCAmelCase=False  , __lowerCAmelCase="<s>"  , __lowerCAmelCase="</s>"  , __lowerCAmelCase="<unk>"  , __lowerCAmelCase="<sep>"  , __lowerCAmelCase="<pad>"  , __lowerCAmelCase="<cls>"  , __lowerCAmelCase="<mask>"  , __lowerCAmelCase=["<eop>", "<eod>"]  , __lowerCAmelCase = None  , **__lowerCAmelCase  , ):
														# Mask token behave like a normal word, i.e. include the space before it
														UpperCamelCase__				     =			AddedToken(__lowerCAmelCase  , lstrip=__lowerCAmelCase  , rstrip=__lowerCAmelCase  ) if isinstance(__lowerCAmelCase  , __lowerCAmelCase  ) else mask_token

														UpperCamelCase__				     =			{} if sp_model_kwargs is None else sp_model_kwargs

														super().__init__(
														    do_lower_case=__lowerCAmelCase  , remove_space=__lowerCAmelCase  , keep_accents=__lowerCAmelCase  , bos_token=__lowerCAmelCase  , eos_token=__lowerCAmelCase  , unk_token=__lowerCAmelCase  , sep_token=__lowerCAmelCase  , pad_token=__lowerCAmelCase  , cls_token=__lowerCAmelCase  , mask_token=__lowerCAmelCase  , additional_special_tokens=__lowerCAmelCase  , sp_model_kwargs=self.sp_model_kwargs  , **__lowerCAmelCase  , )

														UpperCamelCase__				     =			3

														UpperCamelCase__				     =			do_lower_case
														UpperCamelCase__				     =			remove_space
														UpperCamelCase__				     =			keep_accents
														UpperCamelCase__				     =			vocab_file

														UpperCamelCase__				     =			spm.SentencePieceProcessor(**self.sp_model_kwargs  )
														self.sp_model.Load(__lowerCAmelCase  )






							@property
							def 				_lowerCamelCase		(						self  ):
														return len(self.sp_model  )






							def 				_lowerCamelCase		(						self  ):
														UpperCamelCase__				     =			{self.convert_ids_to_tokens(__lowerCAmelCase  ): i for i in range(self.vocab_size  )}
														vocab.update(self.added_tokens_encoder  )
														return vocab






							def __getstate__(						self  ):
														UpperCamelCase__				     =			self.__dict__.copy()
														UpperCamelCase__				     =			None
														return state






							def __setstate__(						self  , __lowerCAmelCase  ):
														UpperCamelCase__				     =			d

														# for backward compatibility
														if not hasattr(self  , """sp_model_kwargs"""  ):
																					UpperCamelCase__				     =			{}

														UpperCamelCase__				     =			spm.SentencePieceProcessor(**self.sp_model_kwargs  )
														self.sp_model.Load(self.vocab_file  )






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  ):
														if self.remove_space:
																					UpperCamelCase__				     =			""" """.join(inputs.strip().split()  )
														else:
																					UpperCamelCase__				     =			inputs
														UpperCamelCase__				     =			outputs.replace("""``"""  , """\""""  ).replace("""''"""  , """\""""  )

														if not self.keep_accents:
																					UpperCamelCase__				     =			unicodedata.normalize("""NFKD"""  , __lowerCAmelCase  )
																					UpperCamelCase__				     =			"""""".join([c for c in outputs if not unicodedata.combining(__lowerCAmelCase  )]  )
														if self.do_lower_case:
																					UpperCamelCase__				     =			outputs.lower()

														return outputs






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  ):
														UpperCamelCase__				     =			self.preprocess_text(__lowerCAmelCase  )
														UpperCamelCase__				     =			self.sp_model.encode(__lowerCAmelCase  , out_type=__lowerCAmelCase  )
														UpperCamelCase__				     =			[]
														for piece in pieces:
																					if len(__lowerCAmelCase  ) > 1 and piece[-1] == str(""","""  ) and piece[-2].isdigit():
																												UpperCamelCase__				     =			self.sp_model.EncodeAsPieces(piece[:-1].replace(__lowerCAmelCase  , """"""  )  )
																												if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																																			if len(cur_pieces[0]  ) == 1:
																																										UpperCamelCase__				     =			cur_pieces[1:]
																																			else:
																																										UpperCamelCase__				     =			cur_pieces[0][1:]
																												cur_pieces.append(piece[-1]  )
																												new_pieces.extend(__lowerCAmelCase  )
																					else:
																												new_pieces.append(__lowerCAmelCase  )

														return new_pieces






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  ):
														return self.sp_model.PieceToId(__lowerCAmelCase  )






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  ):
														return self.sp_model.IdToPiece(__lowerCAmelCase  )






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  ):
														UpperCamelCase__				     =			"""""".join(__lowerCAmelCase  ).replace(__lowerCAmelCase  , """ """  ).strip()
														return out_string






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase = False  , __lowerCAmelCase = None  , __lowerCAmelCase = True  , **__lowerCAmelCase  , ):
														UpperCamelCase__				     =			kwargs.pop("""use_source_tokenizer"""  , __lowerCAmelCase  )

														UpperCamelCase__				     =			self.convert_ids_to_tokens(__lowerCAmelCase  , skip_special_tokens=__lowerCAmelCase  )

														# To avoid mixing byte-level and unicode for byte-level BPT
														# we need to build string separately for added tokens and byte-level tokens
														# cf. https://github.com/huggingface/transformers/issues/1133
														UpperCamelCase__				     =			[]
														UpperCamelCase__				     =			[]
														for token in filtered_tokens:
																					if skip_special_tokens and token in self.all_special_ids:
																												continue
																					if token in self.added_tokens_encoder:
																												if current_sub_text:
																																			sub_texts.append(self.convert_tokens_to_string(__lowerCAmelCase  )  )
																																			UpperCamelCase__				     =			[]
																												sub_texts.append(__lowerCAmelCase  )
																					else:
																												current_sub_text.append(__lowerCAmelCase  )
														if current_sub_text:
																					sub_texts.append(self.convert_tokens_to_string(__lowerCAmelCase  )  )

														# Mimic the behavior of the Rust tokenizer:
														# By default, there are no spaces between special tokens
														UpperCamelCase__				     =			"""""".join(__lowerCAmelCase  )

														UpperCamelCase__				     =			(
														    clean_up_tokenization_spaces
														    if clean_up_tokenization_spaces is not None
														    else self.clean_up_tokenization_spaces
														)
														if clean_up_tokenization_spaces:
																					UpperCamelCase__				     =			self.clean_up_tokenization(__lowerCAmelCase  )
																					return clean_text
														else:
																					return text






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase = None  ):
														UpperCamelCase__				     =			[self.sep_token_id]
														UpperCamelCase__				     =			[self.cls_token_id]
														if token_ids_a is None:
																					return token_ids_a + sep + cls
														return token_ids_a + sep + token_ids_a + sep + cls






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase = None  , __lowerCAmelCase = False  ):

														if already_has_special_tokens:
																					return super().get_special_tokens_mask(
																					    token_ids_a=__lowerCAmelCase  , token_ids_a=__lowerCAmelCase  , already_has_special_tokens=__lowerCAmelCase  )

														if token_ids_a is not None:
																					return ([0] * len(__lowerCAmelCase  )) + [1] + ([0] * len(__lowerCAmelCase  )) + [1, 1]
														return ([0] * len(__lowerCAmelCase  )) + [1, 1]






							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase = None  ):
														UpperCamelCase__				     =			[self.sep_token_id]
														UpperCamelCase__				     =			[2]

														if token_ids_a is None:
																					return len(token_ids_a + sep  ) * [0] + cls_segment_id
														return len(token_ids_a + sep  ) * [0] + len(token_ids_a + sep  ) * [1] + cls_segment_id







							def 				_lowerCamelCase		(						self  , __lowerCAmelCase  , __lowerCAmelCase = None  ):
														if not os.path.isdir(__lowerCAmelCase  ):
																					logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""  )
																					return
														UpperCamelCase__				     =			os.path.join(
														    __lowerCAmelCase  , (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]  )

														if os.path.abspath(self.vocab_file  ) != os.path.abspath(__lowerCAmelCase  ) and os.path.isfile(self.vocab_file  ):
																					copyfile(self.vocab_file  , __lowerCAmelCase  )
														elif not os.path.isfile(self.vocab_file  ):
																					with open(__lowerCAmelCase  , """wb"""  ) as fi:
																												UpperCamelCase__				     =			self.sp_model.serialized_model_proto()
																												fi.write(__lowerCAmelCase  )

														return (out_vocab_file,)









import inspect
import unittest

class        __SCREAMING_SNAKE_CASE					(     unittest.TestCase       ):






							def 				_lowerCamelCase		(						self  ):
														try:
																					import diffusers  # noqa: F401
														except ImportError:
																					assert False







							def 				_lowerCamelCase		(						self  ):
														import diffusers
														from diffusers.dependency_versions_table import deps

														UpperCamelCase__				     =			inspect.getmembers(__lowerCAmelCase  , inspect.isclass  )

														for cls_name, cls_module in all_classes:
																					if "dummy_" in cls_module.__module__:
																												for backend in cls_module._backends:
																																			if backend == "k_diffusion":
																																										UpperCamelCase__				     =			"""k-diffusion"""
																																			elif backend == "invisible_watermark":
																																										UpperCamelCase__				     =			"""invisible-watermark"""
																																			assert backend in deps, f"""{backend} is not in the deps table!"""




