






import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


UpperCAmelCase__      			=						logging.get_logger(__name__)

UpperCAmelCase__      			=						'▁'

UpperCAmelCase__      			=						{'vocab_file': 'vocab.txt', 'sentencepiece_model_ckpt': 'sentencepiece.bpe.model'}

UpperCAmelCase__      			=						{
    'sentencepiece_model_file': 'sentencepiece.bpe.model',
    'vocab_file': 'vocab.txt',
}

UpperCAmelCase__      			=						{
    'vocab_file': {
        'ernie-m-base': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt',
        'ernie-m-large': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt',
    },
    'sentencepiece_model_file': {
        'ernie-m-base': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model',
        'ernie-m-large': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model',
    },
}

UpperCAmelCase__      			=						{
    'ernie-m-base': 514,
    'ernie-m-large': 514,
}

UpperCAmelCase__      			=						{
    'ernie-m-base': {'do_lower_case': False},
    'ernie-m-large': {'do_lower_case': False},
}




class 							snake_case_       (    _SCREAMING_SNAKE_CASE      ):

						"""simple docstring"""



						snake_case__        =      ["""input_ids"""]

						snake_case__        =      VOCAB_FILES_NAMES
						snake_case__        =      PRETRAINED_INIT_CONFIGURATION
						snake_case__        =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						snake_case__        =      PRETRAINED_VOCAB_FILES_MAP
						snake_case__        =      RESOURCE_FILES_NAMES



						def __init__(self:					List[str] ,					__UpperCAmelCase:					Union[str, Any] ,					__UpperCAmelCase:					int=None ,					__UpperCAmelCase:					Tuple=False ,					__UpperCAmelCase:					List[Any]="utf8" ,					__UpperCAmelCase:					Tuple="[UNK]" ,					__UpperCAmelCase:					Tuple="[SEP]" ,					__UpperCAmelCase:					Dict="[PAD]" ,					__UpperCAmelCase:					str="[CLS]" ,					__UpperCAmelCase:					Dict="[MASK]" ,					__UpperCAmelCase:					Optional[Dict[str, Any]] = None ,					**__UpperCAmelCase:					Tuple ,					)    ->     Optional[int]:




											'''simple docstring'''



											__a					:						int      =					{} if sp_model_kwargs is None else sp_model_kwargs
											super().__init__(
											    do_lower_case=_lowerCAmelCase ,					unk_token=_lowerCAmelCase ,					sep_token=_lowerCAmelCase ,					pad_token=_lowerCAmelCase ,					cls_token=_lowerCAmelCase ,					mask_token=_lowerCAmelCase ,					vocab_file=_lowerCAmelCase ,					encoding=_lowerCAmelCase ,					sp_model_kwargs=self.sp_model_kwargs ,					**_lowerCAmelCase ,					)
											__a					:						Optional[Any]      =					do_lower_case
											__a					:						int      =					sentencepiece_model_ckpt
											__a					:						Optional[Any]      =					spm.SentencePieceProcessor(**self.sp_model_kwargs   )
											self.sp_model.Load(_lowerCAmelCase   )

											# to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
											if vocab_file is not None:
																__a					:						Tuple      =					self.load_vocab(filepath=_lowerCAmelCase   )
											else:
																__a					:						Any      =					{self.sp_model.id_to_piece(_lowerCAmelCase   ): id for id in range(self.sp_model.get_piece_size()   )}
											__a					:						Optional[Any]      =					{v: k for k, v in self.vocab.items()}



						def        UpperCAmelCase__					(self:					List[Any] ,					__UpperCAmelCase:					Tuple   )    ->     List[str]:




											'''simple docstring'''



											if text is None:
																return None

											__a					:						Dict      =					self.tokenize(_lowerCAmelCase   )
											__a      ,			__a					:						Dict      =					"", []

											for i, ch in enumerate(_lowerCAmelCase   ):
																if ch in self.SP_CHAR_MAPPING:
																					__a					:						Any      =					self.SP_CHAR_MAPPING.get(_lowerCAmelCase   )
																else:
																					__a					:						Optional[Any]      =					unicodedata.normalize("NFKC" ,					_lowerCAmelCase   )
																if self.is_whitespace(_lowerCAmelCase   ):
																					continue
																normalized_text += ch
																char_mapping.extend([i] * len(_lowerCAmelCase   )   )

											__a      ,			__a      ,			__a					:						Optional[Any]      =					normalized_text, [], 0

											if self.do_lower_case:
																__a					:						Optional[Any]      =					text.lower()

											for token in split_tokens:
																if token[:1] == "▁":
																					__a					:						str      =					token[1:]
																__a					:						Dict      =					text[offset:].index(_lowerCAmelCase   ) + offset
																__a					:						str      =					start + len(_lowerCAmelCase   )

																token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)   )
																__a					:						Optional[Any]      =					end
											return token_mapping



						@property
						def        UpperCAmelCase__					(self:					Optional[Any]   )    ->     Dict:




											'''simple docstring'''



											return len(self.vocab   )



						def        UpperCAmelCase__					(self:					Optional[int]   )    ->     Optional[Any]:




											'''simple docstring'''



											return dict(self.vocab ,					**self.added_tokens_encoder   )



						def __getstate__(self:					List[str]   )    ->     List[Any]:




											'''simple docstring'''



											__a					:						Optional[int]      =					self.__dict__.copy()
											__a					:						str      =					None
											return state



						def __setstate__(self:					int ,					__UpperCAmelCase:					Union[str, Any]   )    ->     Dict:




											'''simple docstring'''



											__a					:						Any      =					d

											# for backward compatibility
											if not hasattr(self ,					"sp_model_kwargs"   ):
																__a					:						Dict      =					{}

											__a					:						List[str]      =					spm.SentencePieceProcessor(**self.sp_model_kwargs   )
											self.sp_model.Load(self.sentencepiece_model_ckpt   )



						def        UpperCAmelCase__					(self:					Tuple ,					__UpperCAmelCase:					List[Any]   )    ->     int:




											'''simple docstring'''



											return "".join((self.SP_CHAR_MAPPING.get(_lowerCAmelCase ,					_lowerCAmelCase   ) for c in text)   )



						def        UpperCAmelCase__					(self:					List[str] ,					__UpperCAmelCase:					List[Any] ,					__UpperCAmelCase:					Optional[int]=False ,					__UpperCAmelCase:					int=64 ,					__UpperCAmelCase:					Tuple=0.1   )    ->     str:




											'''simple docstring'''



											if self.sp_model_kwargs.get("enable_sampling"   ) is True:
																__a					:						Tuple      =					True
											if self.sp_model_kwargs.get("alpha"   ) is not None:
																__a					:						Dict      =					self.sp_model_kwargs.get("alpha"   )
											if self.sp_model_kwargs.get("nbest_size"   ) is not None:
																__a					:						Tuple      =					self.sp_model_kwargs.get("nbest_size"   )

											if not enable_sampling:
																__a					:						Optional[Any]      =					self.sp_model.EncodeAsPieces(_lowerCAmelCase   )
											else:
																__a					:						str      =					self.sp_model.SampleEncodeAsPieces(_lowerCAmelCase ,					_lowerCAmelCase ,					_lowerCAmelCase   )
											__a					:						Tuple      =					[]
											for pi, piece in enumerate(_lowerCAmelCase   ):
																if piece == SPIECE_UNDERLINE:
																					if not pieces[pi + 1].startswith(_lowerCAmelCase   ) and pi != 0:
																										new_pieces.append(_lowerCAmelCase   )
																										continue
																					else:
																										continue
																__a					:						str      =					0
																for i, chunk in enumerate(_lowerCAmelCase   ):
																					if chunk == SPIECE_UNDERLINE:
																										continue
																					if self.is_ch_char(_lowerCAmelCase   ) or self.is_punct(_lowerCAmelCase   ):
																										if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																															new_pieces.append(piece[lst_i:i]   )
																										new_pieces.append(_lowerCAmelCase   )
																										__a					:						Tuple      =					i + 1
																					elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
																										if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																															new_pieces.append(piece[lst_i:i]   )
																										__a					:						int      =					i
																					elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
																										if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
																															new_pieces.append(piece[lst_i:i]   )
																										__a					:						List[Any]      =					i
																if len(_lowerCAmelCase   ) > lst_i:
																					new_pieces.append(piece[lst_i:]   )
											return new_pieces



						def        UpperCAmelCase__					(self:					Union[str, Any] ,					__UpperCAmelCase:					Optional[Any]   )    ->     str:




											'''simple docstring'''



											__a					:						List[Any]      =					"".join(_lowerCAmelCase   ).replace(_lowerCAmelCase ,					" "   ).strip()
											return out_string



						def        UpperCAmelCase__					(self:					Optional[Any] ,					__UpperCAmelCase:					Any   )    ->     Union[str, Any]:




											'''simple docstring'''



											__a					:						int      =					self.convert_ids_to_tokens(_lowerCAmelCase   )
											__a					:						List[str]      =					"".join(_lowerCAmelCase   ).replace(_lowerCAmelCase ,					" "   ).strip()
											return out_string



						def        UpperCAmelCase__					(self:					Union[str, Any] ,					__UpperCAmelCase:					Any   )    ->     Optional[int]:




											'''simple docstring'''



											return self.vocab.get(_lowerCAmelCase ,					self.vocab.get(self.unk_token   )   )



						def        UpperCAmelCase__					(self:					Optional[int] ,					__UpperCAmelCase:					List[str]   )    ->     int:




											'''simple docstring'''



											return self.reverse_vocab.get(_lowerCAmelCase ,					self.unk_token   )



						def        UpperCAmelCase__					(self:					Optional[int] ,					__UpperCAmelCase:					Optional[Any] ,					__UpperCAmelCase:					List[Any]=None   )    ->     Union[str, Any]:




											'''simple docstring'''



											if token_ids_a is None:
																return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
											__a					:						Union[str, Any]      =					[self.cls_token_id]
											__a					:						Optional[Any]      =					[self.sep_token_id]
											return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep



						def        UpperCAmelCase__					(self:					Any ,					__UpperCAmelCase:					Optional[Any] ,					__UpperCAmelCase:					str=None   )    ->     Optional[Any]:




											'''simple docstring'''



											if offset_mapping_a is None:
																return [(0, 0)] + offset_mapping_a + [(0, 0)]

											return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]



						def        UpperCAmelCase__					(self:					Dict ,					__UpperCAmelCase:					List[str] ,					__UpperCAmelCase:					str=None ,					__UpperCAmelCase:					Any=False   )    ->     Dict:




											'''simple docstring'''



											if already_has_special_tokens:
																if token_ids_a is not None:
																					raise ValueError(
																					    "You should not supply a second sequence if the provided sequence of "
																					    "ids is already formatted with special tokens for the model."   )
																return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

											if token_ids_a is not None:
																return [1] + ([0] * len(_lowerCAmelCase   )) + [1, 1] + ([0] * len(_lowerCAmelCase   )) + [1]
											return [1] + ([0] * len(_lowerCAmelCase   )) + [1]



						def        UpperCAmelCase__					(self:					List[str] ,					__UpperCAmelCase:					List[int] ,					__UpperCAmelCase:					Optional[List[int]] = None   )    ->     Any:




											'''simple docstring'''



											if token_ids_a is None:
																# [CLS] X [SEP]
																return (len(_lowerCAmelCase   ) + 2) * [0]

											# [CLS] A [SEP] [SEP] B [SEP]
											return [0] * (len(_lowerCAmelCase   ) + 1) + [1] * (len(_lowerCAmelCase   ) + 3)



						def        UpperCAmelCase__					(self:					List[str] ,					__UpperCAmelCase:					Any   )    ->     Any:




											'''simple docstring'''



											if "\u4e00" <= char <= "\u9fff":
																return True
											return False



						def        UpperCAmelCase__					(self:					List[str] ,					__UpperCAmelCase:					Dict   )    ->     List[str]:




											'''simple docstring'''



											if ("a" <= char <= "z") or ("A" <= char <= "Z"):
																return True
											return False



						def        UpperCAmelCase__					(self:					str ,					__UpperCAmelCase:					Union[str, Any]   )    ->     Tuple:




											'''simple docstring'''



											if char in ",;:.?!~，；：。？！《》【】":
																return True
											return False



						def        UpperCAmelCase__					(self:					Dict ,					__UpperCAmelCase:					Optional[int]   )    ->     Optional[int]:




											'''simple docstring'''



											if char == " " or char == "\t" or char == "\n" or char == "\r":
																return True
											if len(_lowerCAmelCase   ) == 1:
																__a					:						int      =					unicodedata.category(_lowerCAmelCase   )
																if cat == "Zs":
																					return True
											return False



						def        UpperCAmelCase__					(self:					int ,					__UpperCAmelCase:					Tuple   )    ->     Optional[Any]:




											'''simple docstring'''



											__a					:						Any      =					{}
											with io.open(_lowerCAmelCase ,					"r" ,					encoding="utf-8"   ) as f:
																for index, line in enumerate(_lowerCAmelCase   ):
																					__a					:						Dict      =					line.rstrip("\n"   )
																					__a					:						List[Any]      =					int(_lowerCAmelCase   )

											return token_to_idx



						def        UpperCAmelCase__					(self:					str ,					__UpperCAmelCase:					str ,					__UpperCAmelCase:					Optional[str] = None   )    ->     str:




											'''simple docstring'''



											__a					:						Dict      =					0
											if os.path.isdir(_lowerCAmelCase   ):
																__a					:						Union[str, Any]      =					os.path.join(
																    _lowerCAmelCase ,					(filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]   )
											else:
																__a					:						Any      =					(filename_prefix + "-" if filename_prefix else "") + save_directory
											with open(_lowerCAmelCase ,					"w" ,					encoding="utf-8"   ) as writer:
																for token, token_index in sorted(self.vocab.items() ,					key=lambda __UpperCAmelCase   : kv[1]   ):
																					if index != token_index:
																										logger.warning(
																										    f'Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive.'
																										    " Please check that the vocabulary is not corrupted!"   )
																										__a					:						Union[str, Any]      =					token_index
																					writer.write(token + "\n"   )
																					index += 1

											__a					:						List[Any]      =					os.path.join(_lowerCAmelCase ,					"sentencepiece.bpe.model"   )
											with open(_lowerCAmelCase ,					"wb"   ) as fi:
																__a					:						Union[str, Any]      =					self.sp_model.serialized_model_proto()
																fi.write(_lowerCAmelCase   )

											return (vocab_file,)











import unittest
from typing import Tuple

import torch

from diffusers.utils import floats_tensor, randn_tensor, torch_all_close, torch_device
from diffusers.utils.testing_utils import require_torch





@require_torch
class 			lowerCamelCase_				:


 '''simple docstring'''







 @property
 def  lowerCAmelCase_    (				self    :						Optional[Any]		):
       return self.get_dummy_input()



 @property
 def  lowerCAmelCase_    (				self    :						Union[str, Any]		):
       if self.block_type == "down":
             return (4, 32, 16, 16)
       elif self.block_type == "mid":
             return (4, 32, 32, 32)
       elif self.block_type == "up":
             return (4, 32, 64, 64)

       raise ValueError(F"'{self.block_type}' is not a supported block_type. Set it to 'up', 'mid', or 'down'."		)



 def  lowerCAmelCase_    (				self    :						List[str]							,							_lowerCAmelCase    :						Optional[Any]=True							,							_lowerCAmelCase    :						str=False							,							_lowerCAmelCase    :						Optional[int]=False							,							_lowerCAmelCase    :						Dict=False							,							):
       SCREAMING_SNAKE_CASE_      		=     4
       SCREAMING_SNAKE_CASE_      		=     32
       SCREAMING_SNAKE_CASE_      		=     (32, 32)

       SCREAMING_SNAKE_CASE_      		=     torch.manual_seed(0		)
       SCREAMING_SNAKE_CASE_      		=     torch.device(_lowerCAmelCase		)
       SCREAMING_SNAKE_CASE_      		=     (batch_size, num_channels) + sizes
       SCREAMING_SNAKE_CASE_      		=     randn_tensor(_lowerCAmelCase							,							generator=_lowerCAmelCase							,							device=_lowerCAmelCase		)
       SCREAMING_SNAKE_CASE_      		=     {'hidden_states': hidden_states}

       if include_temb:
             SCREAMING_SNAKE_CASE_      		=     128
             SCREAMING_SNAKE_CASE_      		=     randn_tensor((batch_size, temb_channels)							,							generator=_lowerCAmelCase							,							device=_lowerCAmelCase		)

       if include_res_hidden_states_tuple:
             SCREAMING_SNAKE_CASE_      		=     torch.manual_seed(1		)
             SCREAMING_SNAKE_CASE_      		=     (randn_tensor(_lowerCAmelCase							,							generator=_lowerCAmelCase							,							device=_lowerCAmelCase		),)

       if include_encoder_hidden_states:
             SCREAMING_SNAKE_CASE_      		=     floats_tensor((batch_size, 32, 32)		).to(_lowerCAmelCase		)

       if include_skip_sample:
             SCREAMING_SNAKE_CASE_      		=     randn_tensor(((batch_size, 3) + sizes)							,							generator=_lowerCAmelCase							,							device=_lowerCAmelCase		)

       return dummy_input



 def  lowerCAmelCase_    (				self    :						Optional[int]		):
       SCREAMING_SNAKE_CASE_      		=     {
           'in_channels': 32,
           'out_channels': 32,
           'temb_channels': 128,
       }
       if self.block_type == "up":
             SCREAMING_SNAKE_CASE_      		=     32

       if self.block_type == "mid":
             init_dict.pop('out_channels'		)

       SCREAMING_SNAKE_CASE_      		=     self.dummy_input
       return init_dict, inputs_dict



 def  lowerCAmelCase_    (				self    :						Dict							,							_lowerCAmelCase    :						Optional[Any]		):
       SCREAMING_SNAKE_CASE_		,					SCREAMING_SNAKE_CASE_      		=     self.prepare_init_args_and_inputs_for_common()
       SCREAMING_SNAKE_CASE_      		=     self.block_class(**_lowerCAmelCase		)
       unet_block.to(_lowerCAmelCase		)
       unet_block.eval()

       with torch.no_grad():
             SCREAMING_SNAKE_CASE_      		=     unet_block(**_lowerCAmelCase		)

       if isinstance(_lowerCAmelCase							,							_lowerCAmelCase		):
             SCREAMING_SNAKE_CASE_      		=     output[0]

       self.assertEqual(output.shape							,							self.output_shape		)

       SCREAMING_SNAKE_CASE_      		=     output[0, -1, -3:, -3:]
       SCREAMING_SNAKE_CASE_      		=     torch.tensor(_lowerCAmelCase		).to(_lowerCAmelCase		)
       assert torch_all_close(output_slice.flatten()							,							_lowerCAmelCase							,							atol=5E-3		)




 @unittest.skipIf(torch_device == 'mps'							,							'Training is not supported in mps'		)
 def  lowerCAmelCase_    (				self    :						Any		):
       SCREAMING_SNAKE_CASE_		,					SCREAMING_SNAKE_CASE_      		=     self.prepare_init_args_and_inputs_for_common()
       SCREAMING_SNAKE_CASE_      		=     self.block_class(**_lowerCAmelCase		)
       model.to(_lowerCAmelCase		)
       model.train()
       SCREAMING_SNAKE_CASE_      		=     model(**_lowerCAmelCase		)

       if isinstance(_lowerCAmelCase							,							_lowerCAmelCase		):
             SCREAMING_SNAKE_CASE_      		=     output[0]

       SCREAMING_SNAKE_CASE_      		=     torch.device(_lowerCAmelCase		)
       SCREAMING_SNAKE_CASE_      		=     randn_tensor(output.shape							,							device=_lowerCAmelCase		)
       SCREAMING_SNAKE_CASE_      		=     torch.nn.functional.mse_loss(_lowerCAmelCase							,							_lowerCAmelCase		)
       loss.backward()