



"""simple docstring"""





import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCamelCase  			=		logging.get_logger(__name__)

lowerCamelCase  			=		"""▁"""

lowerCamelCase  			=		{"""vocab_file""": """vocab.txt""", """sentencepiece_model_ckpt""": """sentencepiece.bpe.model"""}

lowerCamelCase  			=		{
    """sentencepiece_model_file""": """sentencepiece.bpe.model""",
    """vocab_file""": """vocab.txt""",
}

lowerCamelCase  			=		{
    """vocab_file""": {
        """ernie-m-base""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt""",
        """ernie-m-large""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt""",
    },
    """sentencepiece_model_file""": {
        """ernie-m-base""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model""",
        """ernie-m-large""": """https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model""",
    },
}

lowerCamelCase  			=		{
    """ernie-m-base""": 514,
    """ernie-m-large""": 514,
}

lowerCamelCase  			=		{
    """ernie-m-base""": {"""do_lower_case""": False},
    """ernie-m-large""": {"""do_lower_case""": False},
}
class  lowercase__  (       __snake_case ):







       '''simple docstring'''


       UpperCamelCase		=	['''input_ids''']

       UpperCamelCase		=	VOCAB_FILES_NAMES
       UpperCamelCase		=	PRETRAINED_INIT_CONFIGURATION
       UpperCamelCase		=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
       UpperCamelCase		=	PRETRAINED_VOCAB_FILES_MAP
       UpperCamelCase		=	RESOURCE_FILES_NAMES



       def __init__(				self     :    Optional[Any]						,							_UpperCAmelCase     :    Tuple						,							_UpperCAmelCase     :    Any=None						,							_UpperCAmelCase     :    int=False						,							_UpperCAmelCase     :    str="utf8"						,							_UpperCAmelCase     :    List[Any]="[UNK]"						,							_UpperCAmelCase     :    Union[str, Any]="[SEP]"						,							_UpperCAmelCase     :    Optional[Any]="[PAD]"						,							_UpperCAmelCase     :    List[Any]="[CLS]"						,							_UpperCAmelCase     :    Union[str, Any]="[MASK]"						,							_UpperCAmelCase     :    Optional[Dict[str, Any]] = None						,							**_UpperCAmelCase     :    List[str]						,							)					->		None:


             '''simple docstring'''




             UpperCAmelCase_     					=				{} if sp_model_kwargs is None else sp_model_kwargs
             super().__init__(
                 do_lower_case=__lowerCamelCase						,							unk_token=__lowerCamelCase						,							sep_token=__lowerCamelCase						,							pad_token=__lowerCamelCase						,							cls_token=__lowerCamelCase						,							mask_token=__lowerCamelCase						,							vocab_file=__lowerCamelCase						,							encoding=__lowerCamelCase						,							sp_model_kwargs=self.sp_model_kwargs						,							**__lowerCamelCase						,							)
             UpperCAmelCase_     					=				do_lower_case
             UpperCAmelCase_     					=				sentencepiece_model_ckpt
             UpperCAmelCase_     					=				spm.SentencePieceProcessor(**self.sp_model_kwargs       )
             self.sp_model.Load(__lowerCamelCase       )

             # to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
             if vocab_file is not None:
                   UpperCAmelCase_     					=				self.load_vocab(filepath=__lowerCamelCase       )
             else:
                   UpperCAmelCase_     					=				{self.sp_model.id_to_piece(__lowerCamelCase       ): id for id in range(self.sp_model.get_piece_size()       )}
             UpperCAmelCase_     					=				{v: k for k, v in self.vocab.items()}



       def 	lowercase__							(				self     :    str						,							_UpperCAmelCase     :    Optional[int]       )					->		Tuple:


             '''simple docstring'''




             if text is None:
                   return None

             UpperCAmelCase_     					=				self.tokenize(__lowerCamelCase       )
             UpperCAmelCase_					,       UpperCAmelCase_     					=				"", []

             for i, ch in enumerate(__lowerCamelCase       ):
                   if ch in self.SP_CHAR_MAPPING:
                         UpperCAmelCase_     					=				self.SP_CHAR_MAPPING.get(__lowerCamelCase       )
                   else:
                         UpperCAmelCase_     					=				unicodedata.normalize("NFKC"						,							__lowerCamelCase       )
                   if self.is_whitespace(__lowerCamelCase       ):
                         continue
                   normalized_text += ch
                   char_mapping.extend([i] * len(__lowerCamelCase       )       )

             UpperCAmelCase_					,       UpperCAmelCase_					,       UpperCAmelCase_     					=				normalized_text, [], 0

             if self.do_lower_case:
                   UpperCAmelCase_     					=				text.lower()

             for token in split_tokens:
                   if token[:1] == "▁":
                         UpperCAmelCase_     					=				token[1:]
                   UpperCAmelCase_     					=				text[offset:].index(__lowerCamelCase       ) + offset
                   UpperCAmelCase_     					=				start + len(__lowerCamelCase       )

                   token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)       )
                   UpperCAmelCase_     					=				end
             return token_mapping



       @property
       def 	lowercase__							(				self     :    Optional[Any]       )					->		Dict:


             '''simple docstring'''




             return len(self.vocab       )



       def 	lowercase__							(				self     :    str       )					->		List[str]:


             '''simple docstring'''




             return dict(self.vocab						,							**self.added_tokens_encoder       )



       def __getstate__(				self     :    Optional[Any]       )					->		Any:


             '''simple docstring'''




             UpperCAmelCase_     					=				self.__dict__.copy()
             UpperCAmelCase_     					=				None
             return state



       def __setstate__(				self     :    int						,							_UpperCAmelCase     :    List[Any]       )					->		Optional[int]:


             '''simple docstring'''




             UpperCAmelCase_     					=				d

             # for backward compatibility
             if not hasattr(self						,							"sp_model_kwargs"       ):
                   UpperCAmelCase_     					=				{}

             UpperCAmelCase_     					=				spm.SentencePieceProcessor(**self.sp_model_kwargs       )
             self.sp_model.Load(self.sentencepiece_model_ckpt       )



       def 	lowercase__							(				self     :    Optional[int]						,							_UpperCAmelCase     :    str       )					->		Optional[Any]:


             '''simple docstring'''




             return "".join((self.SP_CHAR_MAPPING.get(__lowerCamelCase						,							__lowerCamelCase       ) for c in text)       )



       def 	lowercase__							(				self     :    List[Any]						,							_UpperCAmelCase     :    List[Any]						,							_UpperCAmelCase     :    Dict=False						,							_UpperCAmelCase     :    int=64						,							_UpperCAmelCase     :    Dict=0.1       )					->		Any:


             '''simple docstring'''




             if self.sp_model_kwargs.get("enable_sampling"       ) is True:
                   UpperCAmelCase_     					=				True
             if self.sp_model_kwargs.get("alpha"       ) is not None:
                   UpperCAmelCase_     					=				self.sp_model_kwargs.get("alpha"       )
             if self.sp_model_kwargs.get("nbest_size"       ) is not None:
                   UpperCAmelCase_     					=				self.sp_model_kwargs.get("nbest_size"       )

             if not enable_sampling:
                   UpperCAmelCase_     					=				self.sp_model.EncodeAsPieces(__lowerCamelCase       )
             else:
                   UpperCAmelCase_     					=				self.sp_model.SampleEncodeAsPieces(__lowerCamelCase						,							__lowerCamelCase						,							__lowerCamelCase       )
             UpperCAmelCase_     					=				[]
             for pi, piece in enumerate(__lowerCamelCase       ):
                   if piece == SPIECE_UNDERLINE:
                         if not pieces[pi + 1].startswith(__lowerCamelCase       ) and pi != 0:
                               new_pieces.append(__lowerCamelCase       )
                               continue
                         else:
                               continue
                   UpperCAmelCase_     					=				0
                   for i, chunk in enumerate(__lowerCamelCase       ):
                         if chunk == SPIECE_UNDERLINE:
                               continue
                         if self.is_ch_char(__lowerCamelCase       ) or self.is_punct(__lowerCamelCase       ):
                               if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                                     new_pieces.append(piece[lst_i:i]       )
                               new_pieces.append(__lowerCamelCase       )
                               UpperCAmelCase_     					=				i + 1
                         elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
                               if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                                     new_pieces.append(piece[lst_i:i]       )
                               UpperCAmelCase_     					=				i
                         elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
                               if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                                     new_pieces.append(piece[lst_i:i]       )
                               UpperCAmelCase_     					=				i
                   if len(__lowerCamelCase       ) > lst_i:
                         new_pieces.append(piece[lst_i:]       )
             return new_pieces



       def 	lowercase__							(				self     :    Union[str, Any]						,							_UpperCAmelCase     :    List[str]       )					->		str:


             '''simple docstring'''




             UpperCAmelCase_     					=				"".join(__lowerCamelCase       ).replace(__lowerCamelCase						,							" "       ).strip()
             return out_string



       def 	lowercase__							(				self     :    Any						,							_UpperCAmelCase     :    Optional[int]       )					->		Union[str, Any]:


             '''simple docstring'''




             UpperCAmelCase_     					=				self.convert_ids_to_tokens(__lowerCamelCase       )
             UpperCAmelCase_     					=				"".join(__lowerCamelCase       ).replace(__lowerCamelCase						,							" "       ).strip()
             return out_string



       def 	lowercase__							(				self     :    Optional[int]						,							_UpperCAmelCase     :    List[Any]       )					->		int:


             '''simple docstring'''




             return self.vocab.get(__lowerCamelCase						,							self.vocab.get(self.unk_token       )       )



       def 	lowercase__							(				self     :    Optional[int]						,							_UpperCAmelCase     :    Optional[Any]       )					->		Optional[Any]:


             '''simple docstring'''




             return self.reverse_vocab.get(__lowerCamelCase						,							self.unk_token       )



       def 	lowercase__							(				self     :    int						,							_UpperCAmelCase     :    Tuple						,							_UpperCAmelCase     :    Optional[Any]=None       )					->		List[str]:


             '''simple docstring'''




             if token_ids_a is None:
                   return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
             UpperCAmelCase_     					=				[self.cls_token_id]
             UpperCAmelCase_     					=				[self.sep_token_id]
             return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep



       def 	lowercase__							(				self     :    Optional[int]						,							_UpperCAmelCase     :    Optional[int]						,							_UpperCAmelCase     :    List[str]=None       )					->		Tuple:


             '''simple docstring'''




             if offset_mapping_a is None:
                   return [(0, 0)] + offset_mapping_a + [(0, 0)]

             return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]



       def 	lowercase__							(				self     :    int						,							_UpperCAmelCase     :    Dict						,							_UpperCAmelCase     :    Optional[Any]=None						,							_UpperCAmelCase     :    Dict=False       )					->		Optional[int]:


             '''simple docstring'''




             if already_has_special_tokens:
                   if token_ids_a is not None:
                         raise ValueError(
                             "You should not supply a second sequence if the provided sequence of "
                             "ids is already formatted with special tokens for the model."       )
                   return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

             if token_ids_a is not None:
                   return [1] + ([0] * len(__lowerCamelCase       )) + [1, 1] + ([0] * len(__lowerCamelCase       )) + [1]
             return [1] + ([0] * len(__lowerCamelCase       )) + [1]



       def 	lowercase__							(				self     :    List[str]						,							_UpperCAmelCase     :    List[int]						,							_UpperCAmelCase     :    Optional[List[int]] = None       )					->		List[int]:


             '''simple docstring'''




             if token_ids_a is None:
                   # [CLS] X [SEP]
                   return (len(__lowerCamelCase       ) + 2) * [0]

             # [CLS] A [SEP] [SEP] B [SEP]
             return [0] * (len(__lowerCamelCase       ) + 1) + [1] * (len(__lowerCamelCase       ) + 3)



       def 	lowercase__							(				self     :    Union[str, Any]						,							_UpperCAmelCase     :    str       )					->		List[str]:


             '''simple docstring'''




             if "\u4e00" <= char <= "\u9fff":
                   return True
             return False



       def 	lowercase__							(				self     :    Optional[Any]						,							_UpperCAmelCase     :    Dict       )					->		Union[str, Any]:


             '''simple docstring'''




             if ("a" <= char <= "z") or ("A" <= char <= "Z"):
                   return True
             return False



       def 	lowercase__							(				self     :    Optional[int]						,							_UpperCAmelCase     :    int       )					->		int:


             '''simple docstring'''




             if char in ",;:.?!~，；：。？！《》【】":
                   return True
             return False



       def 	lowercase__							(				self     :    Optional[Any]						,							_UpperCAmelCase     :    Optional[Any]       )					->		List[str]:


             '''simple docstring'''




             if char == " " or char == "\t" or char == "\n" or char == "\r":
                   return True
             if len(__lowerCamelCase       ) == 1:
                   UpperCAmelCase_     					=				unicodedata.category(__lowerCamelCase       )
                   if cat == "Zs":
                         return True
             return False



       def 	lowercase__							(				self     :    Any						,							_UpperCAmelCase     :    List[Any]       )					->		Union[str, Any]:


             '''simple docstring'''




             UpperCAmelCase_     					=				{}
             with io.open(__lowerCamelCase						,							"r"						,							encoding="utf-8"       ) as f:
                   for index, line in enumerate(__lowerCamelCase       ):
                         UpperCAmelCase_     					=				line.rstrip("\n"       )
                         UpperCAmelCase_     					=				int(__lowerCamelCase       )

             return token_to_idx







       def 	lowercase__							(				self     :    Tuple						,							_UpperCAmelCase     :    str						,							_UpperCAmelCase     :    Optional[str] = None       )					->		Tuple[str]:


             '''simple docstring'''




             UpperCAmelCase_     					=				0
             if os.path.isdir(__lowerCamelCase       ):
                   UpperCAmelCase_     					=				os.path.join(
                       __lowerCamelCase						,							(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]       )
             else:
                   UpperCAmelCase_     					=				(filename_prefix + "-" if filename_prefix else "") + save_directory
             with open(__lowerCamelCase						,							"w"						,							encoding="utf-8"       ) as writer:
                   for token, token_index in sorted(self.vocab.items()						,							key=lambda _UpperCAmelCase       : kv[1]       ):
                         if index != token_index:
                               logger.warning(
                                   F"""Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."""
                                   " Please check that the vocabulary is not corrupted!"       )
                               UpperCAmelCase_     					=				token_index
                         writer.write(token + "\n"       )
                         index += 1

             UpperCAmelCase_     					=				os.path.join(__lowerCamelCase						,							"sentencepiece.bpe.model"       )
             with open(__lowerCamelCase						,							"wb"       ) as fi:
                   UpperCAmelCase_     					=				self.sp_model.serialized_model_proto()
                   fi.write(__lowerCamelCase       )

             return (vocab_file,)








class 	__lowercase					:





      def __init__(		self    :       Optional[int]							)     -> int:






         """simple docstring"""







         UpperCAmelCase      					=  {}





      def 							_lowercase					(		self    :       str							)     -> None:






         """simple docstring"""







         print(self.vertex							)
         for i in self.vertex:
            print(__lowerCamelCase       ,							""" -> """       ,							""" -> """.join([str(__lowerCamelCase							) for j in self.vertex[i]]							)							)





      def 							_lowercase					(		self    :       Optional[Any]       ,							__lowerCamelCase    :       int       ,							__lowerCamelCase    :       int							)     -> None:






         """simple docstring"""







         if from_vertex in self.vertex:
            self.vertex[from_vertex].append(__lowerCamelCase							)
         else:
            # else make a new vertex
            UpperCAmelCase      					=  [to_vertex]





      def 							_lowercase					(		self    :       str							)     -> None:






         """simple docstring"""







         UpperCAmelCase      					=  [False] * len(self.vertex							)

         # call the recursive helper function
         for i in range(len(self.vertex							)							):
            if not visited[i]:
               self.dfs_recursive(__lowerCamelCase       ,							__lowerCamelCase							)





      def 							_lowercase					(		self    :       Optional[int]       ,							__lowerCamelCase    :       int       ,							__lowerCamelCase    :       list							)     -> None:






         """simple docstring"""







         UpperCAmelCase      					=  True

         print(__lowerCamelCase       ,							end=""" """							)

         # Recur for all the vertices that are adjacent to this node
         for i in self.vertex:
            if not visited[i]:
               self.dfs_recursive(__lowerCamelCase       ,							__lowerCamelCase							)


if __name__ == "__main__":
 __a	 =      Graph()
 g.add_edge(0, 1)
 g.add_edge(0, 2)
 g.add_edge(1, 2)
 g.add_edge(2, 0)
 g.add_edge(2, 3)
 g.add_edge(3, 3)

 g.print_graph()
 print("""DFS:""")
 g.dfs()

 # OUTPUT:
 # 0  ->  1 -> 2
 # 1  ->  2
 # 2  ->  0 -> 3
 # 3  ->  3
 # DFS:
 #  0 1 2 3



