




import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


lowerCamelCase__					:  int      							=				logging.get_logger(__name__)

lowerCamelCase__					:  Optional[int]      							=				'▁'

lowerCamelCase__					:  List[Any]      							=				{'vocab_file': 'vocab.txt', 'sentencepiece_model_ckpt': 'sentencepiece.bpe.model'}

lowerCamelCase__					:  Union[str, Any]      							=				{
    'sentencepiece_model_file': 'sentencepiece.bpe.model',
    'vocab_file': 'vocab.txt',
}

lowerCamelCase__					:  List[str]      							=				{
    'vocab_file': {
        'ernie-m-base': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt',
        'ernie-m-large': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt',
    },
    'sentencepiece_model_file': {
        'ernie-m-base': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model',
        'ernie-m-large': 'https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model',
    },
}

lowerCamelCase__					:  Dict      							=				{
    'ernie-m-base': 514,
    'ernie-m-large': 514,
}

lowerCamelCase__					:  List[str]      							=				{
    'ernie-m-base': {'do_lower_case': False},
    'ernie-m-large': {'do_lower_case': False},
}





class 			lowerCamelCase_				( _SCREAMING_SNAKE_CASE   ):


 '''simple docstring'''







 lowercase_							       =       ["input_ids"]

 lowercase_							       =       VOCAB_FILES_NAMES
 lowercase_							       =       PRETRAINED_INIT_CONFIGURATION
 lowercase_							       =       PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
 lowercase_							       =       PRETRAINED_VOCAB_FILES_MAP
 lowercase_							       =       RESOURCE_FILES_NAMES



 def __init__(				self    :						List[str]							,							_lowerCAmelCase    :						Union[str, Any]							,							_lowerCAmelCase    :						int=None							,							_lowerCAmelCase    :						Tuple=False							,							_lowerCAmelCase    :						List[Any]="utf8"							,							_lowerCAmelCase    :						Tuple="[UNK]"							,							_lowerCAmelCase    :						Tuple="[SEP]"							,							_lowerCAmelCase    :						Dict="[PAD]"							,							_lowerCAmelCase    :						str="[CLS]"							,							_lowerCAmelCase    :						Dict="[MASK]"							,							_lowerCAmelCase    :						Optional[Dict[str, Any]] = None							,							**_lowerCAmelCase    :						Tuple							,							):
       # Mask token behave like a normal word, i.e. include the space before it and
       # is included in the raw text, there should be a match in a non-normalized sentence.

       SCREAMING_SNAKE_CASE_      		=     {} if sp_model_kwargs is None else sp_model_kwargs
       super().__init__(
           do_lower_case=_lowerCAmelCase							,							unk_token=_lowerCAmelCase							,							sep_token=_lowerCAmelCase							,							pad_token=_lowerCAmelCase							,							cls_token=_lowerCAmelCase							,							mask_token=_lowerCAmelCase							,							vocab_file=_lowerCAmelCase							,							encoding=_lowerCAmelCase							,							sp_model_kwargs=self.sp_model_kwargs							,							**_lowerCAmelCase							,							)
       SCREAMING_SNAKE_CASE_      		=     do_lower_case
       SCREAMING_SNAKE_CASE_      		=     sentencepiece_model_ckpt
       SCREAMING_SNAKE_CASE_      		=     spm.SentencePieceProcessor(**self.sp_model_kwargs		)
       self.sp_model.Load(_lowerCAmelCase		)

       # to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
       if vocab_file is not None:
             SCREAMING_SNAKE_CASE_      		=     self.load_vocab(filepath=_lowerCAmelCase		)
       else:
             SCREAMING_SNAKE_CASE_      		=     {self.sp_model.id_to_piece(_lowerCAmelCase		): id for id in range(self.sp_model.get_piece_size()		)}
       SCREAMING_SNAKE_CASE_      		=     {v: k for k, v in self.vocab.items()}



 def  lowerCAmelCase_    (				self    :						List[Any]							,							_lowerCAmelCase    :						Tuple		):
       if text is None:
             return None

       SCREAMING_SNAKE_CASE_      		=     self.tokenize(_lowerCAmelCase		)
       SCREAMING_SNAKE_CASE_		,					SCREAMING_SNAKE_CASE_      		=     '', []

       for i, ch in enumerate(_lowerCAmelCase		):
             if ch in self.SP_CHAR_MAPPING:
                   SCREAMING_SNAKE_CASE_      		=     self.SP_CHAR_MAPPING.get(_lowerCAmelCase		)
             else:
                   SCREAMING_SNAKE_CASE_      		=     unicodedata.normalize('NFKC'							,							_lowerCAmelCase		)
             if self.is_whitespace(_lowerCAmelCase		):
                   continue
             normalized_text += ch
             char_mapping.extend([i] * len(_lowerCAmelCase		)		)

       SCREAMING_SNAKE_CASE_		,					SCREAMING_SNAKE_CASE_		,					SCREAMING_SNAKE_CASE_      		=     normalized_text, [], 0

       if self.do_lower_case:
             SCREAMING_SNAKE_CASE_      		=     text.lower()

       for token in split_tokens:
             if token[:1] == "▁":
                   SCREAMING_SNAKE_CASE_      		=     token[1:]
             SCREAMING_SNAKE_CASE_      		=     text[offset:].index(_lowerCAmelCase		) + offset
             SCREAMING_SNAKE_CASE_      		=     start + len(_lowerCAmelCase		)

             token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)		)
             SCREAMING_SNAKE_CASE_      		=     end
       return token_mapping



 @property
 def  lowerCAmelCase_    (				self    :						Optional[Any]		):
       return len(self.vocab		)



 def  lowerCAmelCase_    (				self    :						Optional[int]		):
       return dict(self.vocab							,							**self.added_tokens_encoder		)



 def __getstate__(				self    :						List[str]		):
       SCREAMING_SNAKE_CASE_      		=     self.__dict__.copy()
       SCREAMING_SNAKE_CASE_      		=     None
       return state



 def __setstate__(				self    :						int							,							_lowerCAmelCase    :						Union[str, Any]		):
       SCREAMING_SNAKE_CASE_      		=     d

       # for backward compatibility
       if not hasattr(self							,							'sp_model_kwargs'		):
             SCREAMING_SNAKE_CASE_      		=     {}

       SCREAMING_SNAKE_CASE_      		=     spm.SentencePieceProcessor(**self.sp_model_kwargs		)
       self.sp_model.Load(self.sentencepiece_model_ckpt		)



 def  lowerCAmelCase_    (				self    :						Tuple							,							_lowerCAmelCase    :						List[Any]		):
       return "".join((self.SP_CHAR_MAPPING.get(_lowerCAmelCase							,							_lowerCAmelCase		) for c in text)		)



 def  lowerCAmelCase_    (				self    :						List[str]							,							_lowerCAmelCase    :						List[Any]							,							_lowerCAmelCase    :						Optional[int]=False							,							_lowerCAmelCase    :						int=64							,							_lowerCAmelCase    :						Tuple=0.1		):

       if self.sp_model_kwargs.get('enable_sampling'		) is True:
             SCREAMING_SNAKE_CASE_      		=     True
       if self.sp_model_kwargs.get('alpha'		) is not None:
             SCREAMING_SNAKE_CASE_      		=     self.sp_model_kwargs.get('alpha'		)
       if self.sp_model_kwargs.get('nbest_size'		) is not None:
             SCREAMING_SNAKE_CASE_      		=     self.sp_model_kwargs.get('nbest_size'		)

       if not enable_sampling:
             SCREAMING_SNAKE_CASE_      		=     self.sp_model.EncodeAsPieces(_lowerCAmelCase		)
       else:
             SCREAMING_SNAKE_CASE_      		=     self.sp_model.SampleEncodeAsPieces(_lowerCAmelCase							,							_lowerCAmelCase							,							_lowerCAmelCase		)
       SCREAMING_SNAKE_CASE_      		=     []
       for pi, piece in enumerate(_lowerCAmelCase		):
             if piece == SPIECE_UNDERLINE:
                   if not pieces[pi + 1].startswith(_lowerCAmelCase		) and pi != 0:
                         new_pieces.append(_lowerCAmelCase		)
                         continue
                   else:
                         continue
             SCREAMING_SNAKE_CASE_      		=     0
             for i, chunk in enumerate(_lowerCAmelCase		):
                   if chunk == SPIECE_UNDERLINE:
                         continue
                   if self.is_ch_char(_lowerCAmelCase		) or self.is_punct(_lowerCAmelCase		):
                         if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                               new_pieces.append(piece[lst_i:i]		)
                         new_pieces.append(_lowerCAmelCase		)
                         SCREAMING_SNAKE_CASE_      		=     i + 1
                   elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
                         if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                               new_pieces.append(piece[lst_i:i]		)
                         SCREAMING_SNAKE_CASE_      		=     i
                   elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
                         if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                               new_pieces.append(piece[lst_i:i]		)
                         SCREAMING_SNAKE_CASE_      		=     i
             if len(_lowerCAmelCase		) > lst_i:
                   new_pieces.append(piece[lst_i:]		)
       return new_pieces



 def  lowerCAmelCase_    (				self    :						Union[str, Any]							,							_lowerCAmelCase    :						Optional[Any]		):
       SCREAMING_SNAKE_CASE_      		=     ''.join(_lowerCAmelCase		).replace(_lowerCAmelCase							,							' '		).strip()
       return out_string



 def  lowerCAmelCase_    (				self    :						Optional[Any]							,							_lowerCAmelCase    :						Any		):
       SCREAMING_SNAKE_CASE_      		=     self.convert_ids_to_tokens(_lowerCAmelCase		)
       SCREAMING_SNAKE_CASE_      		=     ''.join(_lowerCAmelCase		).replace(_lowerCAmelCase							,							' '		).strip()
       return out_string



 def  lowerCAmelCase_    (				self    :						Union[str, Any]							,							_lowerCAmelCase    :						Any		):
       return self.vocab.get(_lowerCAmelCase							,							self.vocab.get(self.unk_token		)		)



 def  lowerCAmelCase_    (				self    :						Optional[int]							,							_lowerCAmelCase    :						List[str]		):
       return self.reverse_vocab.get(_lowerCAmelCase							,							self.unk_token		)



 def  lowerCAmelCase_    (				self    :						Optional[int]							,							_lowerCAmelCase    :						Optional[Any]							,							_lowerCAmelCase    :						List[Any]=None		):
       if token_ids_a is None:
             return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
       SCREAMING_SNAKE_CASE_      		=     [self.cls_token_id]
       SCREAMING_SNAKE_CASE_      		=     [self.sep_token_id]
       return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep



 def  lowerCAmelCase_    (				self    :						Any							,							_lowerCAmelCase    :						Optional[Any]							,							_lowerCAmelCase    :						str=None		):
       if offset_mapping_a is None:
             return [(0, 0)] + offset_mapping_a + [(0, 0)]

       return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]



 def  lowerCAmelCase_    (				self    :						Dict							,							_lowerCAmelCase    :						List[str]							,							_lowerCAmelCase    :						str=None							,							_lowerCAmelCase    :						Any=False		):

       if already_has_special_tokens:
             if token_ids_a is not None:
                   raise ValueError(
                       'You should not supply a second sequence if the provided sequence of '
                       'ids is already formatted with special tokens for the model.'		)
             return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

       if token_ids_a is not None:
             return [1] + ([0] * len(_lowerCAmelCase		)) + [1, 1] + ([0] * len(_lowerCAmelCase		)) + [1]
       return [1] + ([0] * len(_lowerCAmelCase		)) + [1]



 def  lowerCAmelCase_    (				self    :						List[str]							,							_lowerCAmelCase    :						List[int]							,							_lowerCAmelCase    :						Optional[List[int]] = None		):
       # called when `add_special_tokens` is True, so align with `build_inputs_with_special_tokens` method
       if token_ids_a is None:
             # [CLS] X [SEP]
             return (len(_lowerCAmelCase		) + 2) * [0]

       # [CLS] A [SEP] [SEP] B [SEP]
       return [0] * (len(_lowerCAmelCase		) + 1) + [1] * (len(_lowerCAmelCase		) + 3)



 def  lowerCAmelCase_    (				self    :						List[str]							,							_lowerCAmelCase    :						Any		):
       if "\u4e00" <= char <= "\u9fff":
             return True
       return False



 def  lowerCAmelCase_    (				self    :						List[str]							,							_lowerCAmelCase    :						Dict		):
       if ("a" <= char <= "z") or ("A" <= char <= "Z"):
             return True
       return False



 def  lowerCAmelCase_    (				self    :						str							,							_lowerCAmelCase    :						Union[str, Any]		):
       if char in ",;:.?!~，；：。？！《》【】":
             return True
       return False



 def  lowerCAmelCase_    (				self    :						Dict							,							_lowerCAmelCase    :						Optional[int]		):
       if char == " " or char == "\t" or char == "\n" or char == "\r":
             return True
       if len(_lowerCAmelCase		) == 1:
             SCREAMING_SNAKE_CASE_      		=     unicodedata.category(_lowerCAmelCase		)
             if cat == "Zs":
                   return True
       return False



 def  lowerCAmelCase_    (				self    :						int							,							_lowerCAmelCase    :						Tuple		):
       SCREAMING_SNAKE_CASE_      		=     {}
       with io.open(_lowerCAmelCase							,							'r'							,							encoding='utf-8'		) as f:
             for index, line in enumerate(_lowerCAmelCase		):
                   SCREAMING_SNAKE_CASE_      		=     line.rstrip('\n'		)
                   SCREAMING_SNAKE_CASE_      		=     int(_lowerCAmelCase		)

       return token_to_idx




 def  lowerCAmelCase_    (				self    :						str							,							_lowerCAmelCase    :						str							,							_lowerCAmelCase    :						Optional[str] = None		):
       SCREAMING_SNAKE_CASE_      		=     0
       if os.path.isdir(_lowerCAmelCase		):
             SCREAMING_SNAKE_CASE_      		=     os.path.join(
                 _lowerCAmelCase							,							(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']		)
       else:
             SCREAMING_SNAKE_CASE_      		=     (filename_prefix + '-' if filename_prefix else '') + save_directory
       with open(_lowerCAmelCase							,							'w'							,							encoding='utf-8'		) as writer:
             for token, token_index in sorted(self.vocab.items()							,							key=lambda _lowerCAmelCase		: kv[1]		):
                   if index != token_index:
                         logger.warning(
                             F"Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."
                             ' Please check that the vocabulary is not corrupted!'		)
                         SCREAMING_SNAKE_CASE_      		=     token_index
                   writer.write(token + '\n'		)
                   index += 1

       SCREAMING_SNAKE_CASE_      		=     os.path.join(_lowerCAmelCase							,							'sentencepiece.bpe.model'		)
       with open(_lowerCAmelCase							,							'wb'		) as fi:
             SCREAMING_SNAKE_CASE_      		=     self.sp_model.serialized_model_proto()
             fi.write(_lowerCAmelCase		)

       return (vocab_file,)





import unittest
from typing import Tuple

import torch

from diffusers.utils import floats_tensor, randn_tensor, torch_all_close, torch_device
from diffusers.utils.testing_utils import require_torch





@require_torch
class 			lowerCamelCase_				:


 '''simple docstring'''







 @property
 def  lowerCAmelCase_    (				self    :						Optional[Any]		):
       return self.get_dummy_input()



 @property
 def  lowerCAmelCase_    (				self    :						Union[str, Any]		):
       if self.block_type == "down":
             return (4, 32, 16, 16)
       elif self.block_type == "mid":
             return (4, 32, 32, 32)
       elif self.block_type == "up":
             return (4, 32, 64, 64)

       raise ValueError(F"'{self.block_type}' is not a supported block_type. Set it to 'up', 'mid', or 'down'."		)



 def  lowerCAmelCase_    (				self    :						List[str]							,							_lowerCAmelCase    :						Optional[Any]=True							,							_lowerCAmelCase    :						str=False							,							_lowerCAmelCase    :						Optional[int]=False							,							_lowerCAmelCase    :						Dict=False							,							):
       SCREAMING_SNAKE_CASE_      		=     4
       SCREAMING_SNAKE_CASE_      		=     32
       SCREAMING_SNAKE_CASE_      		=     (32, 32)

       SCREAMING_SNAKE_CASE_      		=     torch.manual_seed(0		)
       SCREAMING_SNAKE_CASE_      		=     torch.device(_lowerCAmelCase		)
       SCREAMING_SNAKE_CASE_      		=     (batch_size, num_channels) + sizes
       SCREAMING_SNAKE_CASE_      		=     randn_tensor(_lowerCAmelCase							,							generator=_lowerCAmelCase							,							device=_lowerCAmelCase		)
       SCREAMING_SNAKE_CASE_      		=     {'hidden_states': hidden_states}

       if include_temb:
             SCREAMING_SNAKE_CASE_      		=     128
             SCREAMING_SNAKE_CASE_      		=     randn_tensor((batch_size, temb_channels)							,							generator=_lowerCAmelCase							,							device=_lowerCAmelCase		)

       if include_res_hidden_states_tuple:
             SCREAMING_SNAKE_CASE_      		=     torch.manual_seed(1		)
             SCREAMING_SNAKE_CASE_      		=     (randn_tensor(_lowerCAmelCase							,							generator=_lowerCAmelCase							,							device=_lowerCAmelCase		),)

       if include_encoder_hidden_states:
             SCREAMING_SNAKE_CASE_      		=     floats_tensor((batch_size, 32, 32)		).to(_lowerCAmelCase		)

       if include_skip_sample:
             SCREAMING_SNAKE_CASE_      		=     randn_tensor(((batch_size, 3) + sizes)							,							generator=_lowerCAmelCase							,							device=_lowerCAmelCase		)

       return dummy_input



 def  lowerCAmelCase_    (				self    :						Optional[int]		):
       SCREAMING_SNAKE_CASE_      		=     {
           'in_channels': 32,
           'out_channels': 32,
           'temb_channels': 128,
       }
       if self.block_type == "up":
             SCREAMING_SNAKE_CASE_      		=     32

       if self.block_type == "mid":
             init_dict.pop('out_channels'		)

       SCREAMING_SNAKE_CASE_      		=     self.dummy_input
       return init_dict, inputs_dict



 def  lowerCAmelCase_    (				self    :						Dict							,							_lowerCAmelCase    :						Optional[Any]		):
       SCREAMING_SNAKE_CASE_		,					SCREAMING_SNAKE_CASE_      		=     self.prepare_init_args_and_inputs_for_common()
       SCREAMING_SNAKE_CASE_      		=     self.block_class(**_lowerCAmelCase		)
       unet_block.to(_lowerCAmelCase		)
       unet_block.eval()

       with torch.no_grad():
             SCREAMING_SNAKE_CASE_      		=     unet_block(**_lowerCAmelCase		)

       if isinstance(_lowerCAmelCase							,							_lowerCAmelCase		):
             SCREAMING_SNAKE_CASE_      		=     output[0]

       self.assertEqual(output.shape							,							self.output_shape		)

       SCREAMING_SNAKE_CASE_      		=     output[0, -1, -3:, -3:]
       SCREAMING_SNAKE_CASE_      		=     torch.tensor(_lowerCAmelCase		).to(_lowerCAmelCase		)
       assert torch_all_close(output_slice.flatten()							,							_lowerCAmelCase							,							atol=5E-3		)




 @unittest.skipIf(torch_device == 'mps'							,							'Training is not supported in mps'		)
 def  lowerCAmelCase_    (				self    :						Any		):
       SCREAMING_SNAKE_CASE_		,					SCREAMING_SNAKE_CASE_      		=     self.prepare_init_args_and_inputs_for_common()
       SCREAMING_SNAKE_CASE_      		=     self.block_class(**_lowerCAmelCase		)
       model.to(_lowerCAmelCase		)
       model.train()
       SCREAMING_SNAKE_CASE_      		=     model(**_lowerCAmelCase		)

       if isinstance(_lowerCAmelCase							,							_lowerCAmelCase		):
             SCREAMING_SNAKE_CASE_      		=     output[0]

       SCREAMING_SNAKE_CASE_      		=     torch.device(_lowerCAmelCase		)
       SCREAMING_SNAKE_CASE_      		=     randn_tensor(output.shape							,							device=_lowerCAmelCase		)
       SCREAMING_SNAKE_CASE_      		=     torch.nn.functional.mse_loss(_lowerCAmelCase							,							_lowerCAmelCase		)
       loss.backward()