





import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


__A     =    logging.get_logger(__name__)
__A     =    {"vocab_file": "spiece.model"}

__A     =    {
    "vocab_file": {
        "albert-base-v1": "https://huggingface.co/albert-base-v1/resolve/main/spiece.model",
        "albert-large-v1": "https://huggingface.co/albert-large-v1/resolve/main/spiece.model",
        "albert-xlarge-v1": "https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model",
        "albert-xxlarge-v1": "https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model",
        "albert-base-v2": "https://huggingface.co/albert-base-v2/resolve/main/spiece.model",
        "albert-large-v2": "https://huggingface.co/albert-large-v2/resolve/main/spiece.model",
        "albert-xlarge-v2": "https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model",
        "albert-xxlarge-v2": "https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model",
    }
}

__A     =    {
    "albert-base-v1": 512,
    "albert-large-v1": 512,
    "albert-xlarge-v1": 512,
    "albert-xxlarge-v1": 512,
    "albert-base-v2": 512,
    "albert-large-v2": 512,
    "albert-xlarge-v2": 512,
    "albert-xxlarge-v2": 512,
}

__A     =    "▁"



class        _SCREAMING_SNAKE_CASE						(     __SCREAMING_SNAKE_CASE   ):







 '''simple docstring'''
 lowercase_		=      VOCAB_FILES_NAMES
 lowercase_		=      PRETRAINED_VOCAB_FILES_MAP
 lowercase_		=      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES

 def __init__(self    :       Dict	,			UpperCAmelCase_    :       str	,			UpperCAmelCase_    :       Optional[Any]=True	,			UpperCAmelCase_    :       str=True	,			UpperCAmelCase_    :       Union[str, Any]=False	,			UpperCAmelCase_    :       int="[CLS]"	,			UpperCAmelCase_    :       Optional[Any]="[SEP]"	,			UpperCAmelCase_    :       Optional[Any]="<unk>"	,			UpperCAmelCase_    :       int="[SEP]"	,			UpperCAmelCase_    :       List[str]="<pad>"	,			UpperCAmelCase_    :       Union[str, Any]="[CLS]"	,			UpperCAmelCase_    :       str="[MASK]"	,			UpperCAmelCase_    :       Optional[Dict[str, Any]] = None	,			**UpperCAmelCase_    :       Tuple	,			)	->None:



   '''simple docstring'''



   lowerCamelCase__:				Any        =(
       AddedToken(UpperCAmelCase_	,			lstrip=UpperCAmelCase_	,			rstrip=UpperCAmelCase_	,			normalized=UpperCAmelCase_)
       if isinstance(UpperCAmelCase_	,			UpperCAmelCase_)
       else mask_token
   )

   lowerCamelCase__:				Any        ={} if sp_model_kwargs is None else sp_model_kwargs

   super().__init__(
       do_lower_case=UpperCAmelCase_	,			remove_space=UpperCAmelCase_	,			keep_accents=UpperCAmelCase_	,			bos_token=UpperCAmelCase_	,			eos_token=UpperCAmelCase_	,			unk_token=UpperCAmelCase_	,			sep_token=UpperCAmelCase_	,			pad_token=UpperCAmelCase_	,			cls_token=UpperCAmelCase_	,			mask_token=UpperCAmelCase_	,			sp_model_kwargs=self.sp_model_kwargs	,			**UpperCAmelCase_	,			)

   lowerCamelCase__:				Dict        =do_lower_case
   lowerCamelCase__:				List[Any]        =remove_space
   lowerCamelCase__:				Optional[int]        =keep_accents
   lowerCamelCase__:				int        =vocab_file

   lowerCamelCase__:				List[str]        =spm.SentencePieceProcessor(**self.sp_model_kwargs)
   self.sp_model.Load(UpperCAmelCase_)

 @property
 def 							SCREAMING_SNAKE_CASE_ (self    :       Tuple)	->Dict:



   '''simple docstring'''



   return len(self.sp_model)

 def 							SCREAMING_SNAKE_CASE_ (self    :       List[str])	->Optional[Any]:



   '''simple docstring'''



   lowerCamelCase__:				List[str]        ={self.convert_ids_to_tokens(UpperCAmelCase_): i for i in range(self.vocab_size)}
   vocab.update(self.added_tokens_encoder)
   return vocab

 def __getstate__(self    :       Dict)	->Tuple:



   '''simple docstring'''



   lowerCamelCase__:				List[Any]        =self.__dict__.copy()
   lowerCamelCase__:				Union[str, Any]        =None
   return state

 def __setstate__(self    :       Any	,			UpperCAmelCase_    :       Tuple)	->Any:



   '''simple docstring'''



   lowerCamelCase__:				Tuple        =d

   # for backward compatibility
   if not hasattr(self	,			"sp_model_kwargs"):
     lowerCamelCase__:				List[Any]        ={}

   lowerCamelCase__:				Union[str, Any]        =spm.SentencePieceProcessor(**self.sp_model_kwargs)
   self.sp_model.Load(self.vocab_file)

 def 							SCREAMING_SNAKE_CASE_ (self    :       Dict	,			UpperCAmelCase_    :       List[Any])	->Any:



   '''simple docstring'''



   if self.remove_space:
     lowerCamelCase__:				Tuple        =" ".join(inputs.strip().split())
   else:
     lowerCamelCase__:				List[str]        =inputs
   lowerCamelCase__:				str        =outputs.replace("``"	,			"\"").replace("''"	,			"\"")

   if not self.keep_accents:
     lowerCamelCase__:				Union[str, Any]        =unicodedata.normalize("NFKD"	,			UpperCAmelCase_)
     lowerCamelCase__:				Any        ="".join([c for c in outputs if not unicodedata.combining(UpperCAmelCase_)])
   if self.do_lower_case:
     lowerCamelCase__:				Optional[Any]        =outputs.lower()

   return outputs

 def 							SCREAMING_SNAKE_CASE_ (self    :       Union[str, Any]	,			UpperCAmelCase_    :       str)	->List[str]:



   '''simple docstring'''



   lowerCamelCase__:				List[Any]        =self.preprocess_text(UpperCAmelCase_)
   lowerCamelCase__:				Any        =self.sp_model.encode(UpperCAmelCase_	,			out_type=UpperCAmelCase_)
   lowerCamelCase__:				Tuple        =[]
   for piece in pieces:
     if len(UpperCAmelCase_) > 1 and piece[-1] == str(",") and piece[-2].isdigit():
       lowerCamelCase__:				List[str]        =self.sp_model.EncodeAsPieces(piece[:-1].replace(UpperCAmelCase_	,			""))
       if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
         if len(cur_pieces[0]) == 1:
           lowerCamelCase__:				Optional[int]        =cur_pieces[1:]
         else:
           lowerCamelCase__:				List[str]        =cur_pieces[0][1:]
       cur_pieces.append(piece[-1])
       new_pieces.extend(UpperCAmelCase_)
     else:
       new_pieces.append(UpperCAmelCase_)

   return new_pieces

 def 							SCREAMING_SNAKE_CASE_ (self    :       Optional[Any]	,			UpperCAmelCase_    :       List[Any])	->Union[str, Any]:



   '''simple docstring'''



   return self.sp_model.PieceToId(UpperCAmelCase_)

 def 							SCREAMING_SNAKE_CASE_ (self    :       Optional[Any]	,			UpperCAmelCase_    :       List[Any])	->List[str]:



   '''simple docstring'''



   return self.sp_model.IdToPiece(UpperCAmelCase_)

 def 							SCREAMING_SNAKE_CASE_ (self    :       int	,			UpperCAmelCase_    :       Any)	->List[Any]:



   '''simple docstring'''



   lowerCamelCase__:				str        =[]
   lowerCamelCase__:				List[str]        =""
   lowerCamelCase__:				int        =False
   for token in tokens:
     # make sure that special tokens are not decoded using sentencepiece model
     if token in self.all_special_tokens:
       if not prev_is_special:
         out_string += " "
       out_string += self.sp_model.decode(UpperCAmelCase_) + token
       lowerCamelCase__:				Any        =True
       lowerCamelCase__:				Optional[Any]        =[]
     else:
       current_sub_tokens.append(UpperCAmelCase_)
       lowerCamelCase__:				Any        =False
   out_string += self.sp_model.decode(UpperCAmelCase_)
   return out_string.strip()

 def 							SCREAMING_SNAKE_CASE_ (self    :       Any	,			UpperCAmelCase_    :       List[int]	,			UpperCAmelCase_    :       Optional[List[int]] = None)	->List[int]:



   '''simple docstring'''



   lowerCamelCase__:				Union[str, Any]        =[self.sep_token_id]
   lowerCamelCase__:				Any        =[self.cls_token_id]
   if token_ids_a is None:
     return cls + token_ids_a + sep
   return cls + token_ids_a + sep + token_ids_a + sep

 def 							SCREAMING_SNAKE_CASE_ (self    :       Optional[Any]	,			UpperCAmelCase_    :       List[int]	,			UpperCAmelCase_    :       Optional[List[int]] = None	,			UpperCAmelCase_    :       bool = False)	->List[int]:



   '''simple docstring'''



   if already_has_special_tokens:
     return super().get_special_tokens_mask(
         token_ids_a=UpperCAmelCase_	,			token_ids_a=UpperCAmelCase_	,			already_has_special_tokens=UpperCAmelCase_)

   if token_ids_a is not None:
     return [1] + ([0] * len(UpperCAmelCase_)) + [1] + ([0] * len(UpperCAmelCase_)) + [1]
   return [1] + ([0] * len(UpperCAmelCase_)) + [1]

 def 							SCREAMING_SNAKE_CASE_ (self    :       List[str]	,			UpperCAmelCase_    :       List[int]	,			UpperCAmelCase_    :       Optional[List[int]] = None)	->List[int]:



   '''simple docstring'''



   lowerCamelCase__:				Tuple        =[self.sep_token_id]
   lowerCamelCase__:				str        =[self.cls_token_id]

   if token_ids_a is None:
     return len(cls + token_ids_a + sep) * [0]
   return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]







 def 							SCREAMING_SNAKE_CASE_ (self    :       Tuple	,			UpperCAmelCase_    :       str	,			UpperCAmelCase_    :       Optional[str] = None)	->Tuple[str]:



   '''simple docstring'''



   if not os.path.isdir(UpperCAmelCase_):
     logger.error(F"""Vocabulary path ({save_directory}) should be a directory""")
     return
   lowerCamelCase__:				List[str]        =os.path.join(
       UpperCAmelCase_	,			(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"])

   if os.path.abspath(self.vocab_file) != os.path.abspath(UpperCAmelCase_) and os.path.isfile(self.vocab_file):
     copyfile(self.vocab_file	,			UpperCAmelCase_)
   elif not os.path.isfile(self.vocab_file):
     with open(UpperCAmelCase_	,			"wb") as fi:
       lowerCamelCase__:				List[str]        =self.sp_model.serialized_model_proto()
       fi.write(UpperCAmelCase_)

   return (out_vocab_file,)








from __future__ import annotations

from typing import Any



class        _SCREAMING_SNAKE_CASE						:







 '''simple docstring'''

 def __init__(self    :       Any	,			UpperCAmelCase_    :       int)	->None:



   '''simple docstring'''



   lowerCamelCase__:				int        =num_of_nodes
   lowerCamelCase__:				list[list[int]]        =[]
   lowerCamelCase__:				dict[int, int]        ={}

 def 							SCREAMING_SNAKE_CASE_ (self    :       Tuple	,			UpperCAmelCase_    :       int	,			UpperCAmelCase_    :       int	,			UpperCAmelCase_    :       int)	->None:



   '''simple docstring'''



   self.m_edges.append([u_node, v_node, weight])

 def 							SCREAMING_SNAKE_CASE_ (self    :       List[str]	,			UpperCAmelCase_    :       int)	->int:



   '''simple docstring'''



   if self.m_component[u_node] == u_node:
     return u_node
   return self.find_component(self.m_component[u_node])

 def 							SCREAMING_SNAKE_CASE_ (self    :       Union[str, Any]	,			UpperCAmelCase_    :       int)	->None:



   '''simple docstring'''



   if self.m_component[u_node] != u_node:
     for k in self.m_component:
       lowerCamelCase__:				Dict        =self.find_component(UpperCAmelCase_)

 def 							SCREAMING_SNAKE_CASE_ (self    :       Union[str, Any]	,			UpperCAmelCase_    :       list[int]	,			UpperCAmelCase_    :       int	,			UpperCAmelCase_    :       int)	->None:



   '''simple docstring'''



   if component_size[u_node] <= component_size[v_node]:
     lowerCamelCase__:				Optional[int]        =v_node
     component_size[v_node] += component_size[u_node]
     self.set_component(UpperCAmelCase_)

   elif component_size[u_node] >= component_size[v_node]:
     lowerCamelCase__:				Tuple        =self.find_component(UpperCAmelCase_)
     component_size[u_node] += component_size[v_node]
     self.set_component(UpperCAmelCase_)







 def 							SCREAMING_SNAKE_CASE_ (self    :       List[str])	->None:



   '''simple docstring'''



   lowerCamelCase__:				List[Any]        =[]
   lowerCamelCase__:				List[str]        =0

   lowerCamelCase__:				list[Any]        =[-1] * self.m_num_of_nodes

   # A list of components (initialized to all of the nodes)
   for node in range(self.m_num_of_nodes):
     self.m_component.update({node: node})
     component_size.append(1)

   lowerCamelCase__:				List[str]        =self.m_num_of_nodes

   while num_of_components > 1:
     for edge in self.m_edges:
       lowerCamelCase__  , lowerCamelCase__  , lowerCamelCase__:				Tuple        =edge

       lowerCamelCase__:				List[Any]        =self.m_component[u]
       lowerCamelCase__:				str        =self.m_component[v]

       if u_component != v_component:

         for component in (u_component, v_component):
           if (
               minimum_weight_edge[component] == -1
               or minimum_weight_edge[component][2] > w
           ):
             lowerCamelCase__:				Union[str, Any]        =[u, v, w]

     for edge in minimum_weight_edge:
       if isinstance(UpperCAmelCase_	,			UpperCAmelCase_):
         lowerCamelCase__  , lowerCamelCase__  , lowerCamelCase__:				Tuple        =edge

         lowerCamelCase__:				str        =self.m_component[u]
         lowerCamelCase__:				Any        =self.m_component[v]

         if u_component != v_component:
           mst_weight += w
           self.union(UpperCAmelCase_	,			UpperCAmelCase_	,			UpperCAmelCase_)
           print(F"""Added edge [{u} - {v}]\nAdded weight: {w}\n""")
           num_of_components -= 1

     lowerCamelCase__:				Tuple        =[-1] * self.m_num_of_nodes
   print(F"""The total weight of the minimal spanning tree is: {mst_weight}""")





def 					lowerCAmelCase_			(	)					->		None:







  """simple docstring"""







if __name__ == "__main__":
     import doctest

     doctest.testmod()

