





import io
import os
import unicodedata
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import PreTrainedTokenizer
from ...utils import logging


_lowerCAmelCase				:   Tuple						       =						logging.get_logger(__name__)

_lowerCAmelCase				:   Tuple						       =						'''▁'''

_lowerCAmelCase				:   str						       =						{'''vocab_file''': '''vocab.txt''', '''sentencepiece_model_ckpt''': '''sentencepiece.bpe.model'''}

_lowerCAmelCase				:   Optional[int]						       =						{
    '''sentencepiece_model_file''': '''sentencepiece.bpe.model''',
    '''vocab_file''': '''vocab.txt''',
}

_lowerCAmelCase				:   List[Any]						       =						{
    '''vocab_file''': {
        '''ernie-m-base''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt''',
        '''ernie-m-large''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/vocab.txt''',
    },
    '''sentencepiece_model_file''': {
        '''ernie-m-base''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model''',
        '''ernie-m-large''': '''https://huggingface.co/susnato/ernie-m-base_pytorch/blob/main/sentencepiece.bpe.model''',
    },
}

_lowerCAmelCase				:   List[Any]						       =						{
    '''ernie-m-base''': 514,
    '''ernie-m-large''': 514,
}

_lowerCAmelCase				:   Optional[int]						       =						{
    '''ernie-m-base''': {'''do_lower_case''': False},
    '''ernie-m-large''': {'''do_lower_case''': False},
}


class        __magic_name__	(				lowerCamelCase__				):





     """simple docstring"""



     __UpperCamelCase            =  ["input_ids"]

     __UpperCamelCase            =  VOCAB_FILES_NAMES
     __UpperCamelCase            =  PRETRAINED_INIT_CONFIGURATION
     __UpperCamelCase            =  PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __UpperCamelCase            =  PRETRAINED_VOCAB_FILES_MAP
     __UpperCamelCase            =  RESOURCE_FILES_NAMES



     def __init__(				self							:str      ,   snake_case							:List[Any]      ,   snake_case							:List[str]=None      ,   snake_case							:Dict=False      ,   snake_case							:Optional[Any]="utf8"      ,   snake_case							:str="[UNK]"      ,   snake_case							:Union[str, Any]="[SEP]"      ,   snake_case							:str="[PAD]"      ,   snake_case							:Any="[CLS]"      ,   snake_case							:int="[MASK]"      ,   snake_case							:Optional[Dict[str, Any]] = None      ,   **snake_case							:List[str]      ,   ):





        '''simple docstring'''




        A_       :					Tuple        =      {} if sp_model_kwargs is None else sp_model_kwargs
        super().__init__(
            do_lower_case=snake_case      ,   unk_token=snake_case      ,   sep_token=snake_case      ,   pad_token=snake_case      ,   cls_token=snake_case      ,   mask_token=snake_case      ,   vocab_file=snake_case      ,   encoding=snake_case      ,   sp_model_kwargs=self.sp_model_kwargs      ,   **snake_case      ,   )
        A_       :					int        =      do_lower_case
        A_       :					str        =      sentencepiece_model_ckpt
        A_       :					Any        =      spm.SentencePieceProcessor(**self.sp_model_kwargs	)
        self.sp_model.Load(snake_case	)

        # to mimic paddlenlp.transformers.ernie_m.tokenizer.ErnieMTokenizer functioning
        if vocab_file is not None:
           A_       :					Union[str, Any]        =      self.load_vocab(filepath=snake_case	)
        else:
           A_       :					List[str]        =      {self.sp_model.id_to_piece(snake_case	): id for id in range(self.sp_model.get_piece_size()	)}
        A_       :					Any        =      {v: k for k, v in self.vocab.items()}



     def     SCREAMING_SNAKE_CASE		(				self							:int      ,   snake_case							:Optional[Any]	):





        '''simple docstring'''




        if text is None:
           return None

        A_       :					List[Any]        =      self.tokenize(snake_case	)
        A_   ,    A_       :					Dict        =      "", []

        for i, ch in enumerate(snake_case	):
           if ch in self.SP_CHAR_MAPPING:
              A_       :					Dict        =      self.SP_CHAR_MAPPING.get(snake_case	)
           else:
              A_       :					Any        =      unicodedata.normalize("NFKC"      ,   snake_case	)
           if self.is_whitespace(snake_case	):
              continue
           normalized_text += ch
           char_mapping.extend([i] * len(snake_case	)	)

        A_   ,    A_   ,    A_       :					Any        =      normalized_text, [], 0

        if self.do_lower_case:
           A_       :					Optional[int]        =      text.lower()

        for token in split_tokens:
           if token[:1] == "▁":
              A_       :					Tuple        =      token[1:]
           A_       :					List[Any]        =      text[offset:].index(snake_case	) + offset
           A_       :					Union[str, Any]        =      start + len(snake_case	)

           token_mapping.append((char_mapping[start], char_mapping[end - 1] + 1)	)
           A_       :					Optional[int]        =      end
        return token_mapping



     @property
     def     SCREAMING_SNAKE_CASE		(				self							:Optional[int]	):





        '''simple docstring'''




        return len(self.vocab	)



     def     SCREAMING_SNAKE_CASE		(				self							:List[str]	):





        '''simple docstring'''




        return dict(self.vocab      ,   **self.added_tokens_encoder	)



     def __getstate__(				self							:Tuple	):





        '''simple docstring'''




        A_       :					str        =      self.__dict__.copy()
        A_       :					List[Any]        =      None
        return state



     def __setstate__(				self							:int      ,   snake_case							:str	):





        '''simple docstring'''




        A_       :					str        =      d

        # for backward compatibility
        if not hasattr(self      ,   "sp_model_kwargs"	):
           A_       :					List[str]        =      {}

        A_       :					Dict        =      spm.SentencePieceProcessor(**self.sp_model_kwargs	)
        self.sp_model.Load(self.sentencepiece_model_ckpt	)



     def     SCREAMING_SNAKE_CASE		(				self							:str      ,   snake_case							:List[str]	):





        '''simple docstring'''




        return "".join((self.SP_CHAR_MAPPING.get(snake_case      ,   snake_case	) for c in text)	)



     def     SCREAMING_SNAKE_CASE		(				self							:int      ,   snake_case							:Union[str, Any]      ,   snake_case							:str=False      ,   snake_case							:Union[str, Any]=64      ,   snake_case							:str=0.1	):





        '''simple docstring'''




        if self.sp_model_kwargs.get("enable_sampling"	) is True:
           A_       :					Union[str, Any]        =      True
        if self.sp_model_kwargs.get("alpha"	) is not None:
           A_       :					Union[str, Any]        =      self.sp_model_kwargs.get("alpha"	)
        if self.sp_model_kwargs.get("nbest_size"	) is not None:
           A_       :					List[Any]        =      self.sp_model_kwargs.get("nbest_size"	)

        if not enable_sampling:
           A_       :					Union[str, Any]        =      self.sp_model.EncodeAsPieces(snake_case	)
        else:
           A_       :					Union[str, Any]        =      self.sp_model.SampleEncodeAsPieces(snake_case      ,   snake_case      ,   snake_case	)
        A_       :					int        =      []
        for pi, piece in enumerate(snake_case	):
           if piece == SPIECE_UNDERLINE:
              if not pieces[pi + 1].startswith(snake_case	) and pi != 0:
                 new_pieces.append(snake_case	)
                 continue
              else:
                 continue
           A_       :					int        =      0
           for i, chunk in enumerate(snake_case	):
              if chunk == SPIECE_UNDERLINE:
                 continue
              if self.is_ch_char(snake_case	) or self.is_punct(snake_case	):
                 if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                    new_pieces.append(piece[lst_i:i]	)
                 new_pieces.append(snake_case	)
                 A_       :					Any        =      i + 1
              elif chunk.isdigit() and i > 0 and not piece[i - 1].isdigit():
                 if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                    new_pieces.append(piece[lst_i:i]	)
                 A_       :					str        =      i
              elif not chunk.isdigit() and i > 0 and piece[i - 1].isdigit():
                 if i > lst_i and piece[lst_i:i] != SPIECE_UNDERLINE:
                    new_pieces.append(piece[lst_i:i]	)
                 A_       :					List[Any]        =      i
           if len(snake_case	) > lst_i:
              new_pieces.append(piece[lst_i:]	)
        return new_pieces



     def     SCREAMING_SNAKE_CASE		(				self							:Tuple      ,   snake_case							:str	):





        '''simple docstring'''




        A_       :					Any        =      "".join(snake_case	).replace(snake_case      ,   " "	).strip()
        return out_string



     def     SCREAMING_SNAKE_CASE		(				self							:Union[str, Any]      ,   snake_case							:int	):





        '''simple docstring'''




        A_       :					int        =      self.convert_ids_to_tokens(snake_case	)
        A_       :					Optional[Any]        =      "".join(snake_case	).replace(snake_case      ,   " "	).strip()
        return out_string



     def     SCREAMING_SNAKE_CASE		(				self							:List[Any]      ,   snake_case							:Any	):





        '''simple docstring'''




        return self.vocab.get(snake_case      ,   self.vocab.get(self.unk_token	)	)



     def     SCREAMING_SNAKE_CASE		(				self							:str      ,   snake_case							:Optional[int]	):





        '''simple docstring'''




        return self.reverse_vocab.get(snake_case      ,   self.unk_token	)



     def     SCREAMING_SNAKE_CASE		(				self							:Optional[Any]      ,   snake_case							:List[Any]      ,   snake_case							:str=None	):





        '''simple docstring'''




        if token_ids_a is None:
           return [self.cls_token_id] + token_ids_a + [self.sep_token_id]
        A_       :					List[Any]        =      [self.cls_token_id]
        A_       :					List[Any]        =      [self.sep_token_id]
        return _cls + token_ids_a + _sep + _sep + token_ids_a + _sep



     def     SCREAMING_SNAKE_CASE		(				self							:Optional[int]      ,   snake_case							:Optional[int]      ,   snake_case							:List[str]=None	):





        '''simple docstring'''




        if offset_mapping_a is None:
           return [(0, 0)] + offset_mapping_a + [(0, 0)]

        return [(0, 0)] + offset_mapping_a + [(0, 0), (0, 0)] + offset_mapping_a + [(0, 0)]



     def     SCREAMING_SNAKE_CASE		(				self							:Optional[int]      ,   snake_case							:Optional[int]      ,   snake_case							:List[Any]=None      ,   snake_case							:int=False	):





        '''simple docstring'''




        if already_has_special_tokens:
           if token_ids_a is not None:
              raise ValueError(
                  "You should not supply a second sequence if the provided sequence of "
                  "ids is already formatted with special tokens for the model."	)
           return [1 if x in [self.sep_token_id, self.cls_token_id] else 0 for x in token_ids_a]

        if token_ids_a is not None:
           return [1] + ([0] * len(snake_case	)) + [1, 1] + ([0] * len(snake_case	)) + [1]
        return [1] + ([0] * len(snake_case	)) + [1]



     def     SCREAMING_SNAKE_CASE		(				self							:List[str]      ,   snake_case							:List[int]      ,   snake_case							:Optional[List[int]] = None	):





        '''simple docstring'''




        if token_ids_a is None:
           # [CLS] X [SEP]
           return (len(snake_case	) + 2) * [0]

        # [CLS] A [SEP] [SEP] B [SEP]
        return [0] * (len(snake_case	) + 1) + [1] * (len(snake_case	) + 3)



     def     SCREAMING_SNAKE_CASE		(				self							:str      ,   snake_case							:Optional[int]	):





        '''simple docstring'''




        if "\u4e00" <= char <= "\u9fff":
           return True
        return False



     def     SCREAMING_SNAKE_CASE		(				self							:str      ,   snake_case							:str	):





        '''simple docstring'''




        if ("a" <= char <= "z") or ("A" <= char <= "Z"):
           return True
        return False



     def     SCREAMING_SNAKE_CASE		(				self							:Optional[Any]      ,   snake_case							:Any	):





        '''simple docstring'''




        if char in ",;:.?!~，；：。？！《》【】":
           return True
        return False



     def     SCREAMING_SNAKE_CASE		(				self							:Any      ,   snake_case							:Optional[Any]	):





        '''simple docstring'''




        if char == " " or char == "\t" or char == "\n" or char == "\r":
           return True
        if len(snake_case	) == 1:
           A_       :					Tuple        =      unicodedata.category(snake_case	)
           if cat == "Zs":
              return True
        return False



     def     SCREAMING_SNAKE_CASE		(				self							:List[str]      ,   snake_case							:Union[str, Any]	):





        '''simple docstring'''




        A_       :					Tuple        =      {}
        with io.open(snake_case      ,   "r"      ,   encoding="utf-8"	) as f:
           for index, line in enumerate(snake_case	):
              A_       :					Optional[int]        =      line.rstrip("\n"	)
              A_       :					Union[str, Any]        =      int(snake_case	)

        return token_to_idx





     def     SCREAMING_SNAKE_CASE		(				self							:Dict      ,   snake_case							:str      ,   snake_case							:Optional[str] = None	):





        '''simple docstring'''




        A_       :					Union[str, Any]        =      0
        if os.path.isdir(snake_case	):
           A_       :					str        =      os.path.join(
               snake_case      ,   (filename_prefix + "-" if filename_prefix else "") + VOCAB_FILES_NAMES["vocab_file"]	)
        else:
           A_       :					Optional[int]        =      (filename_prefix + "-" if filename_prefix else "") + save_directory
        with open(snake_case      ,   "w"      ,   encoding="utf-8"	) as writer:
           for token, token_index in sorted(self.vocab.items()      ,   key=lambda snake_case	: kv[1]	):
              if index != token_index:
                 logger.warning(
                     f"Saving vocabulary to {vocab_file}: vocabulary indices are not consecutive."
                     " Please check that the vocabulary is not corrupted!"	)
                 A_       :					str        =      token_index
              writer.write(token + "\n"	)
              index += 1

        A_       :					Optional[Any]        =      os.path.join(snake_case      ,   "sentencepiece.bpe.model"	)
        with open(snake_case      ,   "wb"	) as fi:
           A_       :					Optional[int]        =      self.sp_model.serialized_model_proto()
           fi.write(snake_case	)

        return (vocab_file,)







import random
import unittest

import torch

from diffusers import IFInpaintingPipeline
from diffusers.utils import floats_tensor
from diffusers.utils.import_utils import is_xformers_available
from diffusers.utils.testing_utils import skip_mps, torch_device

from ..pipeline_params import (
    TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS,
    TEXT_GUIDED_IMAGE_INPAINTING_PARAMS,
)
from ..test_pipelines_common import PipelineTesterMixin
from . import IFPipelineTesterMixin


@skip_mps
class        __magic_name__	(				lowerCamelCase__     ,					lowerCamelCase__     ,					unittest.TestCase				):





     """simple docstring"""



     __UpperCamelCase            =  IFInpaintingPipeline
     __UpperCamelCase            =  TEXT_GUIDED_IMAGE_INPAINTING_PARAMS - {'''width''', '''height'''}
     __UpperCamelCase            =  TEXT_GUIDED_IMAGE_INPAINTING_BATCH_PARAMS
     __UpperCamelCase            =  PipelineTesterMixin.required_optional_params - {'''latents'''}



     def     SCREAMING_SNAKE_CASE		(				self							:Optional[Any]	):





        '''simple docstring'''




        return self._get_dummy_components()



     def     SCREAMING_SNAKE_CASE		(				self							:List[Any]      ,   snake_case							:str      ,   snake_case							:List[Any]=0	):





        '''simple docstring'''




        if str(snake_case	).startswith("mps"	):
           A_       :					Optional[Any]        =      torch.manual_seed(snake_case	)
        else:
           A_       :					Any        =      torch.Generator(device=snake_case	).manual_seed(snake_case	)

        A_       :					Optional[Any]        =      floats_tensor((1, 3, 32, 32)      ,   rng=random.Random(snake_case	)	).to(snake_case	)
        A_       :					Any        =      floats_tensor((1, 3, 32, 32)      ,   rng=random.Random(snake_case	)	).to(snake_case	)

        A_       :					List[Any]        =      {
            "prompt": "A painting of a squirrel eating a burger",
            "image": image,
            "mask_image": mask_image,
            "generator": generator,
            "num_inference_steps": 2,
            "output_type": "numpy",
        }

        return inputs



     @unittest.skipIf(
         torch_device != "cuda" or not is_xformers_available()      ,   reason="XFormers attention is only available with CUDA and `xformers` installed"      ,   )
     def     SCREAMING_SNAKE_CASE		(				self							:Any	):





        '''simple docstring'''




        self._test_xformers_attention_forwardGenerator_pass(expected_max_diff=1e-3	)



     def     SCREAMING_SNAKE_CASE		(				self							:Optional[int]	):





        '''simple docstring'''




        self._test_save_load_optional_components()



     @unittest.skipIf(torch_device != "cuda"      ,   reason="float16 requires CUDA"	)
     def     SCREAMING_SNAKE_CASE		(				self							:Union[str, Any]	):





        '''simple docstring'''




        super().test_save_load_floataa(expected_max_diff=1e-1	)



     def     SCREAMING_SNAKE_CASE		(				self							:Dict	):





        '''simple docstring'''




        self._test_attention_slicing_forward_pass(expected_max_diff=1e-2	)



     def     SCREAMING_SNAKE_CASE		(				self							:List[Any]	):





        '''simple docstring'''




        self._test_save_load_local()





     def     SCREAMING_SNAKE_CASE		(				self							:Dict	):





        '''simple docstring'''




        self._test_inference_batch_single_identical(
            expected_max_diff=1e-2      ,   )
