





import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


_UpperCAmelCase       : Dict			     =       logging.get_logger(__name__)
_UpperCAmelCase       : Dict			     =       {"""vocab_file""": """spiece.model"""}

_UpperCAmelCase       : Union[str, Any]			     =       {
    """vocab_file""": {
        """albert-base-v1""": """https://huggingface.co/albert-base-v1/resolve/main/spiece.model""",
        """albert-large-v1""": """https://huggingface.co/albert-large-v1/resolve/main/spiece.model""",
        """albert-xlarge-v1""": """https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model""",
        """albert-xxlarge-v1""": """https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model""",
        """albert-base-v2""": """https://huggingface.co/albert-base-v2/resolve/main/spiece.model""",
        """albert-large-v2""": """https://huggingface.co/albert-large-v2/resolve/main/spiece.model""",
        """albert-xlarge-v2""": """https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model""",
        """albert-xxlarge-v2""": """https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model""",
    }
}

_UpperCAmelCase       : str			     =       {
    """albert-base-v1""": 5_12,
    """albert-large-v1""": 5_12,
    """albert-xlarge-v1""": 5_12,
    """albert-xxlarge-v1""": 5_12,
    """albert-base-v2""": 5_12,
    """albert-large-v2""": 5_12,
    """albert-xlarge-v2""": 5_12,
    """albert-xxlarge-v2""": 5_12,
}

_UpperCAmelCase       : int			     =       """▁"""

class      lowerCAmelCase							(      __UpperCamelCase    ):
 UpperCAmelCase__									=	VOCAB_FILES_NAMES
 UpperCAmelCase__									=	PRETRAINED_VOCAB_FILES_MAP
 UpperCAmelCase__									=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 def __init__(   self						:   Optional[int]    ,  UpperCAmelCase						:   Dict    ,  UpperCAmelCase						:   Tuple=True    ,  UpperCAmelCase						:   int=True    ,  UpperCAmelCase						:   Any=False    ,  UpperCAmelCase						:   Dict="[CLS]"    ,  UpperCAmelCase						:   List[str]="[SEP]"    ,  UpperCAmelCase						:   Dict="<unk>"    ,  UpperCAmelCase						:   Tuple="[SEP]"    ,  UpperCAmelCase						:   str="<pad>"    ,  UpperCAmelCase						:   Union[str, Any]="[CLS]"    ,  UpperCAmelCase						:   Tuple="[MASK]"    ,  UpperCAmelCase						:   Optional[Dict[str, Any]] = None    ,  **UpperCAmelCase						:   Optional[int]    ,  )    ->			None:
     # Mask token behave like a normal word, i.e. include the space before it and
     # is included in the raw text, there should be a match in a non-normalized sentence.
     lowerCamelCase__					:   int      				=	(
         AddedToken(UpperCAmelCase    ,  lstrip=UpperCAmelCase    ,  rstrip=UpperCAmelCase    ,  normalized=UpperCAmelCase			)
         if isinstance(UpperCAmelCase    ,  UpperCAmelCase			)
         else mask_token
     )

     lowerCamelCase__					:   int      				=	{} if sp_model_kwargs is None else sp_model_kwargs

     super().__init__(
         do_lower_case=UpperCAmelCase    ,  remove_space=UpperCAmelCase    ,  keep_accents=UpperCAmelCase    ,  bos_token=UpperCAmelCase    ,  eos_token=UpperCAmelCase    ,  unk_token=UpperCAmelCase    ,  sep_token=UpperCAmelCase    ,  pad_token=UpperCAmelCase    ,  cls_token=UpperCAmelCase    ,  mask_token=UpperCAmelCase    ,  sp_model_kwargs=self.sp_model_kwargs    ,  **UpperCAmelCase    ,  )

     lowerCamelCase__					:   int      				=	do_lower_case
     lowerCamelCase__					:   Dict      				=	remove_space
     lowerCamelCase__					:   Dict      				=	keep_accents
     lowerCamelCase__					:   Optional[int]      				=	vocab_file

     lowerCamelCase__					:   Optional[int]      				=	spm.SentencePieceProcessor(**self.sp_model_kwargs			)
     self.sp_model.Load(UpperCAmelCase			)
 @property
 def   A_			(   self						:   Optional[Any]			)    ->			Optional[Any]:
     return len(self.sp_model			)
 def   A_			(   self						:   Optional[Any]			)    ->			Union[str, Any]:
     lowerCamelCase__					:   Union[str, Any]      				=	{self.convert_ids_to_tokens(UpperCAmelCase			): i for i in range(self.vocab_size			)}
     vocab.update(self.added_tokens_encoder			)
     return vocab
 def __getstate__(   self						:   Tuple			)    ->			Union[str, Any]:
     lowerCamelCase__					:   Optional[Any]      				=	self.__dict__.copy()
     lowerCamelCase__					:   Any      				=	None
     return state
 def __setstate__(   self						:   int    ,  UpperCAmelCase						:   List[str]			)    ->			Optional[Any]:
     lowerCamelCase__					:   Optional[Any]      				=	d

     # for backward compatibility
     if not hasattr(self    ,  'sp_model_kwargs'			):
         lowerCamelCase__					:   int      				=	{}

     lowerCamelCase__					:   int      				=	spm.SentencePieceProcessor(**self.sp_model_kwargs			)
     self.sp_model.Load(self.vocab_file			)
 def   A_			(   self						:   Optional[int]    ,  UpperCAmelCase						:   Dict			)    ->			str:
     if self.remove_space:
         lowerCamelCase__					:   Optional[Any]      				=	' '.join(inputs.strip().split()			)
     else:
         lowerCamelCase__					:   List[str]      				=	inputs
     lowerCamelCase__					:   Optional[int]      				=	outputs.replace('``'    ,  '"'			).replace('\'\''    ,  '"'			)

     if not self.keep_accents:
         lowerCamelCase__					:   Tuple      				=	unicodedata.normalize('NFKD'    ,  UpperCAmelCase			)
         lowerCamelCase__					:   List[Any]      				=	''.join([c for c in outputs if not unicodedata.combining(UpperCAmelCase			)]			)
     if self.do_lower_case:
         lowerCamelCase__					:   Tuple      				=	outputs.lower()

     return outputs
 def   A_			(   self						:   List[Any]    ,  UpperCAmelCase						:   str			)    ->			List[str]:
     lowerCamelCase__					:   str      				=	self.preprocess_text(UpperCAmelCase			)
     lowerCamelCase__					:   Dict      				=	self.sp_model.encode(UpperCAmelCase    ,  out_type=UpperCAmelCase			)
     lowerCamelCase__					:   Union[str, Any]      				=	[]
     for piece in pieces:
         if len(UpperCAmelCase			) > 1 and piece[-1] == str(','			) and piece[-2].isdigit():
             lowerCamelCase__					:   int      				=	self.sp_model.EncodeAsPieces(piece[:-1].replace(UpperCAmelCase    ,  ''			)			)
             if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
                 if len(cur_pieces[0]			) == 1:
                     lowerCamelCase__					:   int      				=	cur_pieces[1:]
                 else:
                     lowerCamelCase__					:   Optional[Any]      				=	cur_pieces[0][1:]
             cur_pieces.append(piece[-1]			)
             new_pieces.extend(UpperCAmelCase			)
         else:
             new_pieces.append(UpperCAmelCase			)

     return new_pieces
 def   A_			(   self						:   int    ,  UpperCAmelCase						:   Dict			)    ->			int:
     return self.sp_model.PieceToId(UpperCAmelCase			)
 def   A_			(   self						:   Dict    ,  UpperCAmelCase						:   Any			)    ->			Optional[Any]:
     return self.sp_model.IdToPiece(UpperCAmelCase			)
 def   A_			(   self						:   List[Any]    ,  UpperCAmelCase						:   List[Any]			)    ->			List[str]:
     lowerCamelCase__					:   Optional[int]      				=	[]
     lowerCamelCase__					:   str      				=	''
     lowerCamelCase__					:   str      				=	False
     for token in tokens:
         # make sure that special tokens are not decoded using sentencepiece model
         if token in self.all_special_tokens:
             if not prev_is_special:
                 out_string += " "
             out_string += self.sp_model.decode(UpperCAmelCase			) + token
             lowerCamelCase__					:   List[str]      				=	True
             lowerCamelCase__					:   List[Any]      				=	[]
         else:
             current_sub_tokens.append(UpperCAmelCase			)
             lowerCamelCase__					:   Optional[Any]      				=	False
     out_string += self.sp_model.decode(UpperCAmelCase			)
     return out_string.strip()
 def   A_			(   self						:   Tuple    ,  UpperCAmelCase						:   List[int]    ,  UpperCAmelCase						:   Optional[List[int]] = None			)    ->			List[int]:
     lowerCamelCase__					:   Dict      				=	[self.sep_token_id]
     lowerCamelCase__					:   Dict      				=	[self.cls_token_id]
     if token_ids_a is None:
         return cls + token_ids_a + sep
     return cls + token_ids_a + sep + token_ids_a + sep
 def   A_			(   self						:   str    ,  UpperCAmelCase						:   List[int]    ,  UpperCAmelCase						:   Optional[List[int]] = None    ,  UpperCAmelCase						:   bool = False			)    ->			List[int]:

     if already_has_special_tokens:
         return super().get_special_tokens_mask(
             token_ids_a=UpperCAmelCase    ,  token_ids_a=UpperCAmelCase    ,  already_has_special_tokens=UpperCAmelCase			)

     if token_ids_a is not None:
         return [1] + ([0] * len(UpperCAmelCase			)) + [1] + ([0] * len(UpperCAmelCase			)) + [1]
     return [1] + ([0] * len(UpperCAmelCase			)) + [1]
 def   A_			(   self						:   Union[str, Any]    ,  UpperCAmelCase						:   List[int]    ,  UpperCAmelCase						:   Optional[List[int]] = None			)    ->			List[int]:
     lowerCamelCase__					:   Tuple      				=	[self.sep_token_id]
     lowerCamelCase__					:   str      				=	[self.cls_token_id]

     if token_ids_a is None:
         return len(cls + token_ids_a + sep			) * [0]
     return len(cls + token_ids_a + sep			) * [0] + len(token_ids_a + sep			) * [1]







 def   A_			(   self						:   Tuple    ,  UpperCAmelCase						:   str    ,  UpperCAmelCase						:   Optional[str] = None			)    ->			Tuple[str]:
     if not os.path.isdir(UpperCAmelCase			):
         logger.error(F"""Vocabulary path ({save_directory}) should be a directory"""			)
         return
     lowerCamelCase__					:   str      				=	os.path.join(
         UpperCAmelCase    ,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']			)

     if os.path.abspath(self.vocab_file			) != os.path.abspath(UpperCAmelCase			) and os.path.isfile(self.vocab_file			):
         copyfile(self.vocab_file    ,  UpperCAmelCase			)
     elif not os.path.isfile(self.vocab_file			):
         with open(UpperCAmelCase    ,  'wb'			) as fi:
             lowerCamelCase__					:   Optional[int]      				=	self.sp_model.serialized_model_proto()
             fi.write(UpperCAmelCase			)

     return (out_vocab_file,)












class      lowerCAmelCase							:
 def __init__(   self						:   Union[str, Any]    ,  UpperCAmelCase						:   list			)    ->			None:
     lowerCamelCase__					:   int      				=	set_counts
     lowerCamelCase__					:   List[str]      				=	max(UpperCAmelCase			)
     lowerCamelCase__					:   Dict      				=	len(UpperCAmelCase			)
     lowerCamelCase__					:   List[str]      				=	[1] * num_sets
     lowerCamelCase__					:   List[str]      				=	list(range(UpperCAmelCase			)			)
 def   A_			(   self						:   Dict    ,  UpperCAmelCase						:   int    ,  UpperCAmelCase						:   int			)    ->			bool:
     lowerCamelCase__					:   List[str]      				=	self.get_parent(UpperCAmelCase			)
     lowerCamelCase__					:   List[str]      				=	self.get_parent(UpperCAmelCase			)

     if src_parent == dst_parent:
         return False

     if self.ranks[dst_parent] >= self.ranks[src_parent]:
         self.set_counts[dst_parent] += self.set_counts[src_parent]
         lowerCamelCase__					:   Union[str, Any]      				=	0
         lowerCamelCase__					:   str      				=	dst_parent
         if self.ranks[dst_parent] == self.ranks[src_parent]:
             self.ranks[dst_parent] += 1
         lowerCamelCase__					:   int      				=	self.set_counts[dst_parent]
     else:
         self.set_counts[src_parent] += self.set_counts[dst_parent]
         lowerCamelCase__					:   List[str]      				=	0
         lowerCamelCase__					:   Tuple      				=	src_parent
         lowerCamelCase__					:   List[str]      				=	self.set_counts[src_parent]

     lowerCamelCase__					:   Optional[int]      				=	max(self.max_set    ,  UpperCAmelCase			)
     return True







 def   A_			(   self						:   str    ,  UpperCAmelCase						:   int			)    ->			int:
     if self.parents[disj_set] == disj_set:
         return disj_set
     lowerCamelCase__					:   Dict      				=	self.get_parent(self.parents[disj_set]			)
     return self.parents[disj_set]





