






"""simple docstring"""
import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_lowerCAmelCase       =				logging.get_logger(__name__)
_lowerCAmelCase       =				{"vocab_file": "spiece.model"}

_lowerCAmelCase       =				{
    "vocab_file": {
        "albert-base-v1": "https://huggingface.co/albert-base-v1/resolve/main/spiece.model",
        "albert-large-v1": "https://huggingface.co/albert-large-v1/resolve/main/spiece.model",
        "albert-xlarge-v1": "https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model",
        "albert-xxlarge-v1": "https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model",
        "albert-base-v2": "https://huggingface.co/albert-base-v2/resolve/main/spiece.model",
        "albert-large-v2": "https://huggingface.co/albert-large-v2/resolve/main/spiece.model",
        "albert-xlarge-v2": "https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model",
        "albert-xxlarge-v2": "https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model",
    }
}

_lowerCAmelCase       =				{
    "albert-base-v1": 512,
    "albert-large-v1": 512,
    "albert-xlarge-v1": 512,
    "albert-xxlarge-v1": 512,
    "albert-base-v2": 512,
    "albert-large-v2": 512,
    "albert-xlarge-v2": 512,
    "albert-xxlarge-v2": 512,
}

_lowerCAmelCase       =				"▁"



class   SCREAMING_SNAKE_CASE	(					_SCREAMING_SNAKE_CASE    ):





	"""simple docstring"""




	_A							:					Any 			=					VOCAB_FILES_NAMES
	_A							:					str 			=					PRETRAINED_VOCAB_FILES_MAP
	_A							:					int 			=					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES





	def __init__(self  ,  lowerCAmelCase_  ,  lowerCAmelCase_=True  ,  lowerCAmelCase_=True  ,  lowerCAmelCase_=False  ,  lowerCAmelCase_="[CLS]"  ,  lowerCAmelCase_="[SEP]"  ,  lowerCAmelCase_="<unk>"  ,  lowerCAmelCase_="[SEP]"  ,  lowerCAmelCase_="<pad>"  ,  lowerCAmelCase_="[CLS]"  ,  lowerCAmelCase_="[MASK]"  ,  lowerCAmelCase_ = None  ,  **lowerCAmelCase_  ,  ):
		# Mask token behave like a normal word, i.e. include the space before it and
		# is included in the raw text, there should be a match in a non-normalized sentence.
		A_		:    Tuple						     = (
		    AddedToken(lowerCAmelCase_  ,  lstrip=lowerCAmelCase_  ,  rstrip=lowerCAmelCase_  ,  normalized=lowerCAmelCase_				)
		    if isinstance(lowerCAmelCase_  ,  lowerCAmelCase_				)
		    else mask_token
		)

		A_		:    List[str]						     = {} if sp_model_kwargs is None else sp_model_kwargs

		super().__init__(
		    do_lower_case=lowerCAmelCase_  ,  remove_space=lowerCAmelCase_  ,  keep_accents=lowerCAmelCase_  ,  bos_token=lowerCAmelCase_  ,  eos_token=lowerCAmelCase_  ,  unk_token=lowerCAmelCase_  ,  sep_token=lowerCAmelCase_  ,  pad_token=lowerCAmelCase_  ,  cls_token=lowerCAmelCase_  ,  mask_token=lowerCAmelCase_  ,  sp_model_kwargs=self.sp_model_kwargs  ,  **lowerCAmelCase_  ,  )

		A_		:    Optional[Any]						     = do_lower_case
		A_		:    Optional[Any]						     = remove_space
		A_		:    int						     = keep_accents
		A_		:    Tuple						     = vocab_file

		A_		:    Any						     = spm.SentencePieceProcessor(**self.sp_model_kwargs				)
		self.sp_model.Load(lowerCAmelCase_				)





	@property
	def  lowerCamelCase(self				):
		return len(self.sp_model				)





	def  lowerCamelCase(self				):
		A_		:    Optional[int]						     = {self.convert_ids_to_tokens(lowerCAmelCase_				): i for i in range(self.vocab_size				)}
		vocab.update(self.added_tokens_encoder				)
		return vocab





	def __getstate__(self				):
		A_		:    Tuple						     = self.__dict__.copy()
		A_		:    Optional[int]						     = None
		return state





	def __setstate__(self  ,  lowerCAmelCase_				):
		A_		:    Any						     = d

		# for backward compatibility
		if not hasattr(self  ,  """sp_model_kwargs"""				):
			A_		:    Any						     = {}

		A_		:    List[str]						     = spm.SentencePieceProcessor(**self.sp_model_kwargs				)
		self.sp_model.Load(self.vocab_file				)





	def  lowerCamelCase(self  ,  lowerCAmelCase_				):
		if self.remove_space:
			A_		:    Optional[int]						     = """ """.join(inputs.strip().split()				)
		else:
			A_		:    List[str]						     = inputs
		A_		:    Optional[Any]						     = outputs.replace("""``"""  ,  """\""""				).replace("""''"""  ,  """\""""				)

		if not self.keep_accents:
			A_		:    List[Any]						     = unicodedata.normalize("""NFKD"""  ,  lowerCAmelCase_				)
			A_		:    Optional[int]						     = """""".join([c for c in outputs if not unicodedata.combining(lowerCAmelCase_				)]				)
		if self.do_lower_case:
			A_		:    List[str]						     = outputs.lower()

		return outputs





	def  lowerCamelCase(self  ,  lowerCAmelCase_				):
		A_		:    Optional[Any]						     = self.preprocess_text(lowerCAmelCase_				)
		A_		:    Dict						     = self.sp_model.encode(lowerCAmelCase_  ,  out_type=lowerCAmelCase_				)
		A_		:    Any						     = []
		for piece in pieces:
			if len(lowerCAmelCase_				) > 1 and piece[-1] == str(""","""				) and piece[-2].isdigit():
				A_		:    Optional[int]						     = self.sp_model.EncodeAsPieces(piece[:-1].replace(lowerCAmelCase_  ,  """"""				)				)
				if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
					if len(cur_pieces[0]				) == 1:
						A_		:    Any						     = cur_pieces[1:]
					else:
						A_		:    Tuple						     = cur_pieces[0][1:]
				cur_pieces.append(piece[-1]				)
				new_pieces.extend(lowerCAmelCase_				)
			else:
				new_pieces.append(lowerCAmelCase_				)

		return new_pieces





	def  lowerCamelCase(self  ,  lowerCAmelCase_				):
		return self.sp_model.PieceToId(lowerCAmelCase_				)





	def  lowerCamelCase(self  ,  lowerCAmelCase_				):
		return self.sp_model.IdToPiece(lowerCAmelCase_				)





	def  lowerCamelCase(self  ,  lowerCAmelCase_				):
		A_		:    str						     = []
		A_		:    Union[str, Any]						     = """"""
		A_		:    Union[str, Any]						     = False
		for token in tokens:
			# make sure that special tokens are not decoded using sentencepiece model
			if token in self.all_special_tokens:
				if not prev_is_special:
					out_string += " "
				out_string += self.sp_model.decode(lowerCAmelCase_				) + token
				A_		:    str						     = True
				A_		:    Tuple						     = []
			else:
				current_sub_tokens.append(lowerCAmelCase_				)
				A_		:    Union[str, Any]						     = False
		out_string += self.sp_model.decode(lowerCAmelCase_				)
		return out_string.strip()





	def  lowerCamelCase(self  ,  lowerCAmelCase_  ,  lowerCAmelCase_ = None				):
		A_		:    Optional[Any]						     = [self.sep_token_id]
		A_		:    Dict						     = [self.cls_token_id]
		if token_ids_a is None:
			return cls + token_ids_a + sep
		return cls + token_ids_a + sep + token_ids_a + sep





	def  lowerCamelCase(self  ,  lowerCAmelCase_  ,  lowerCAmelCase_ = None  ,  lowerCAmelCase_ = False				):

		if already_has_special_tokens:
			return super().get_special_tokens_mask(
			    token_ids_a=lowerCAmelCase_  ,  token_ids_a=lowerCAmelCase_  ,  already_has_special_tokens=lowerCAmelCase_				)

		if token_ids_a is not None:
			return [1] + ([0] * len(lowerCAmelCase_				)) + [1] + ([0] * len(lowerCAmelCase_				)) + [1]
		return [1] + ([0] * len(lowerCAmelCase_				)) + [1]





	def  lowerCamelCase(self  ,  lowerCAmelCase_  ,  lowerCAmelCase_ = None				):
		A_		:    List[str]						     = [self.sep_token_id]
		A_		:    Optional[Any]						     = [self.cls_token_id]

		if token_ids_a is None:
			return len(cls + token_ids_a + sep				) * [0]
		return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]





	def  lowerCamelCase(self  ,  lowerCAmelCase_  ,  lowerCAmelCase_ = None				):
		if not os.path.isdir(lowerCAmelCase_				):
			logger.error(f"""Vocabulary path ({save_directory}) should be a directory"""				)
			return
		A_		:    Optional[Any]						     = os.path.join(
		    lowerCAmelCase_  ,  (filename_prefix + """-""" if filename_prefix else """""") + VOCAB_FILES_NAMES["""vocab_file"""]				)

		if os.path.abspath(self.vocab_file				) != os.path.abspath(lowerCAmelCase_				) and os.path.isfile(self.vocab_file				):
			copyfile(self.vocab_file  ,  lowerCAmelCase_				)
		elif not os.path.isfile(self.vocab_file				):
			with open(lowerCAmelCase_  ,  """wb"""				) as fi:
				A_		:    List[str]						     = self.sp_model.serialized_model_proto()
				fi.write(lowerCAmelCase_				)

		return (out_vocab_file,)









"""simple docstring"""
class   SCREAMING_SNAKE_CASE	:





	"""simple docstring"""




	def __init__(self  ,  lowerCAmelCase_  ,  lowerCAmelCase_=None  ,  lowerCAmelCase_=None				):
		A_		:    Dict						     = data
		A_		:    Dict						     = previous
		A_		:    int						     = next_node





	def __str__(self				):
		return f"""{self.data}"""





	def  lowerCamelCase(self				):
		return self.data





	def  lowerCamelCase(self				):
		return self.next





	def  lowerCamelCase(self				):
		return self.previous






class   SCREAMING_SNAKE_CASE	:





	"""simple docstring"""




	def __init__(self  ,  lowerCAmelCase_				):
		A_		:    Optional[Any]						     = head





	def __iter__(self				):
		return self





	def  lowerCamelCase(self				):
		if not self.current:
			raise StopIteration
		else:
			A_		:    str						     = self.current.get_data()
			A_		:    List[str]						     = self.current.get_next()
			return value






class   SCREAMING_SNAKE_CASE	:





	"""simple docstring"""




	def __init__(self				):
		A_		:    str						     = None  # First node in list
		A_		:    Tuple						     = None  # Last node in list





	def __str__(self				):
		A_		:    int						     = self.head
		A_		:    str						     = []
		while current is not None:
			nodes.append(current.get_data()				)
			A_		:    Any						     = current.get_next()
		return " ".join(str(lowerCAmelCase_				) for node in nodes				)





	def __contains__(self  ,  lowerCAmelCase_				):
		A_		:    Union[str, Any]						     = self.head
		while current:
			if current.get_data() == value:
				return True
			A_		:    Union[str, Any]						     = current.get_next()
		return False





	def __iter__(self				):
		return LinkedListIterator(self.head				)





	def  lowerCamelCase(self				):
		if self.head:
			return self.head.get_data()
		return None





	def  lowerCamelCase(self				):
		if self.tail:
			return self.tail.get_data()
		return None





	def  lowerCamelCase(self  ,  lowerCAmelCase_				):
		if self.head is None:
			A_		:    str						     = node
			A_		:    Optional[int]						     = node
		else:
			self.insert_before_node(self.head  ,  lowerCAmelCase_				)





	def  lowerCamelCase(self  ,  lowerCAmelCase_				):
		if self.head is None:
			self.set_head(lowerCAmelCase_				)
		else:
			self.insert_after_node(self.tail  ,  lowerCAmelCase_				)





	def  lowerCamelCase(self  ,  lowerCAmelCase_				):
		A_		:    List[Any]						     = Node(lowerCAmelCase_				)
		if self.head is None:
			self.set_head(lowerCAmelCase_				)
		else:
			self.set_tail(lowerCAmelCase_				)





	def  lowerCamelCase(self  ,  lowerCAmelCase_  ,  lowerCAmelCase_				):
		A_		:    str						     = node
		A_		:    Union[str, Any]						     = node.previous

		if node.get_previous() is None:
			A_		:    Dict						     = node_to_insert
		else:
			A_		:    Optional[Any]						     = node_to_insert

		A_		:    Any						     = node_to_insert





	def  lowerCamelCase(self  ,  lowerCAmelCase_  ,  lowerCAmelCase_				):
		A_		:    Dict						     = node
		A_		:    Optional[Any]						     = node.next

		if node.get_next() is None:
			A_		:    Optional[int]						     = node_to_insert
		else:
			A_		:    List[Any]						     = node_to_insert

		A_		:    Optional[Any]						     = node_to_insert





	def  lowerCamelCase(self  ,  lowerCAmelCase_  ,  lowerCAmelCase_				):
		A_		:    str						     = 1
		A_		:    Tuple						     = Node(lowerCAmelCase_				)
		A_		:    Dict						     = self.head
		while node:
			if current_position == position:
				self.insert_before_node(lowerCAmelCase_  ,  lowerCAmelCase_				)
				return
			current_position += 1
			A_		:    List[str]						     = node.next
		self.insert_after_node(self.tail  ,  lowerCAmelCase_				)





	def  lowerCamelCase(self  ,  lowerCAmelCase_				):
		A_		:    Dict						     = self.head
		while node:
			if node.get_data() == item:
				return node
			A_		:    Optional[int]						     = node.get_next()
		raise Exception("""Node not found"""				)





	def  lowerCamelCase(self  ,  lowerCAmelCase_				):
		if (node := self.get_node(lowerCAmelCase_				)) is not None:
			if node == self.head:
				A_		:    Tuple						     = self.head.get_next()

			if node == self.tail:
				A_		:    Dict						     = self.tail.get_previous()

			self.remove_node_pointers(lowerCAmelCase_				)





	@staticmethod
	def  lowerCamelCase(lowerCAmelCase_				):
		if node.get_next():
			A_		:    Any						     = node.previous

		if node.get_previous():
			A_		:    int						     = node.next

		A_		:    str						     = None
		A_		:    int						     = None





	def  lowerCamelCase(self				):
		return self.head is None






def       __UpperCamelCase						(		):
	pass


if __name__ == "__main__":
							import doctest

							doctest.testmod()

