import json
from typing import List, Optional, Tuple

from tokenizers import normalizers
from tokenizers.pre_tokenizers import BertPreTokenizer, PreTokenizer

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_roformer import RoFormerTokenizer
from .tokenization_utils import JiebaPreTokenizer


lowercase_     :   Tuple 				=				logging.get_logger(__name__)

lowercase_     :   Union[str, Any] 				=				{'vocab_file': 'vocab.txt', 'tokenizer_file': 'tokenizer.json'}

lowercase_     :   str 				=				{
    'vocab_file': {
        'junnyu/roformer_chinese_small': 'https://huggingface.co/junnyu/roformer_chinese_small/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_base': 'https://huggingface.co/junnyu/roformer_chinese_base/resolve/main/vocab.txt',
        'junnyu/roformer_chinese_char_small': (
            'https://huggingface.co/junnyu/roformer_chinese_char_small/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_chinese_char_base': (
            'https://huggingface.co/junnyu/roformer_chinese_char_base/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_discriminator': (
            'https://huggingface.co/junnyu/roformer_small_discriminator/resolve/main/vocab.txt'
        ),
        'junnyu/roformer_small_generator': (
            'https://huggingface.co/junnyu/roformer_small_generator/resolve/main/vocab.txt'
        ),
    }
}

lowercase_     :   Tuple 				=				{
    'junnyu/roformer_chinese_small': 1_5_3_6,
    'junnyu/roformer_chinese_base': 1_5_3_6,
    'junnyu/roformer_chinese_char_small': 5_1_2,
    'junnyu/roformer_chinese_char_base': 5_1_2,
    'junnyu/roformer_small_discriminator': 1_2_8,
    'junnyu/roformer_small_generator': 1_2_8,
}


lowercase_     :   Any 				=				{
    'junnyu/roformer_chinese_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_base': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_small': {'do_lower_case': True},
    'junnyu/roformer_chinese_char_base': {'do_lower_case': True},
    'junnyu/roformer_small_discriminator': {'do_lower_case': True},
    'junnyu/roformer_small_generator': {'do_lower_case': True},
}






class 		_lowerCamelCase		(   UpperCamelCase_    ):
    __a  =      VOCAB_FILES_NAMES
    __a  =      PRETRAINED_VOCAB_FILES_MAP
    __a  =      PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
    __a  =      PRETRAINED_INIT_CONFIGURATION
    __a  =      RoFormerTokenizer
    def __init__(  self , lowerCAmelCase=None , lowerCAmelCase=None , lowerCAmelCase=True , lowerCAmelCase="[UNK]" , lowerCAmelCase="[SEP]" , lowerCAmelCase="[PAD]" , lowerCAmelCase="[CLS]" , lowerCAmelCase="[MASK]" , lowerCAmelCase=True , lowerCAmelCase=None , **lowerCAmelCase , )       ->					Union[str, Any]:
     super().__init__(
         lowerCAmelCase , tokenizer_file=lowerCAmelCase , do_lower_case=lowerCAmelCase , unk_token=lowerCAmelCase , sep_token=lowerCAmelCase , pad_token=lowerCAmelCase , cls_token=lowerCAmelCase , mask_token=lowerCAmelCase , tokenize_chinese_chars=lowerCAmelCase , strip_accents=lowerCAmelCase , **lowerCAmelCase , )

     SCREAMING_SNAKE_CASE__:   List[Any]=      json.loads(self.backend_tokenizer.normalizer.__getstate__()   )
     if (
         pre_tok_state.get('''lowercase''' , lowerCAmelCase   ) != do_lower_case
         or pre_tok_state.get('''strip_accents''' , lowerCAmelCase   ) != strip_accents
     ):
      SCREAMING_SNAKE_CASE__:   Optional[Any]=      getattr(lowerCAmelCase , pre_tok_state.pop('''type'''   )   )
      SCREAMING_SNAKE_CASE__:   Union[str, Any]=      do_lower_case
      SCREAMING_SNAKE_CASE__:   Optional[Any]=      strip_accents
      SCREAMING_SNAKE_CASE__:   List[str]=      pre_tok_class(**lowerCAmelCase   )

     SCREAMING_SNAKE_CASE__:   Any=      do_lower_case
    def __getstate__(  self   )       ->					str:
     SCREAMING_SNAKE_CASE__:   Any=      self.__dict__.copy()
     SCREAMING_SNAKE_CASE__:   str=      BertPreTokenizer()
     return state
    def __setstate__(  self , lowerCAmelCase   )       ->					Tuple:
     SCREAMING_SNAKE_CASE__:   Union[str, Any]=      d
     SCREAMING_SNAKE_CASE__:   Tuple=      self.__dict__['''_tokenizer'''].get_vocab()
     SCREAMING_SNAKE_CASE__:   str=      PreTokenizer.custom(JiebaPreTokenizer(lowerCAmelCase   )   )
    def 						UpperCamelCase_				(  self , lowerCAmelCase , lowerCAmelCase=None   )       ->					Optional[Any]:
     SCREAMING_SNAKE_CASE__:   Tuple=      [self.cls_token_id] + token_ids_a + [self.sep_token_id]

     if token_ids_a:
      output += token_ids_a + [self.sep_token_id]

     return output
    def 						UpperCamelCase_				(  self , lowerCAmelCase , lowerCAmelCase = None   )       ->					List[int]:
     SCREAMING_SNAKE_CASE__:   Any=      [self.sep_token_id]
     SCREAMING_SNAKE_CASE__:   List[Any]=      [self.cls_token_id]
     if token_ids_a is None:
      return len(cls + token_ids_a + sep   ) * [0]
     return len(cls + token_ids_a + sep   ) * [0] + len(token_ids_a + sep   ) * [1]
    def 						UpperCamelCase_				(  self , lowerCAmelCase , lowerCAmelCase = None   )       ->					Tuple[str]:
     SCREAMING_SNAKE_CASE__:   Optional[int]=      self._tokenizer.model.save(lowerCAmelCase , name=lowerCAmelCase   )
     return tuple(lowerCAmelCase   )



    def 						UpperCamelCase_				(  self , lowerCAmelCase , lowerCAmelCase=None , lowerCAmelCase=None , lowerCAmelCase=False , **lowerCAmelCase , )       ->					Dict:
     SCREAMING_SNAKE_CASE__:   Optional[int]=      BertPreTokenizer()
     return super().save_pretrained(lowerCAmelCase , lowerCAmelCase , lowerCAmelCase , lowerCAmelCase , **lowerCAmelCase   )


import unittest

from diffusers.pipelines.pipeline_utils import is_safetensors_compatible






class 		_lowerCamelCase		(   unittest.TestCase    ):
    def 						UpperCamelCase_				(  self   )       ->					str:
     SCREAMING_SNAKE_CASE__:   List[Any]=      [
         '''safety_checker/pytorch_model.bin''',
         '''safety_checker/model.safetensors''',
         '''vae/diffusion_pytorch_model.bin''',
         '''vae/diffusion_pytorch_model.safetensors''',
         '''text_encoder/pytorch_model.bin''',
         '''text_encoder/model.safetensors''',
         '''unet/diffusion_pytorch_model.bin''',
         '''unet/diffusion_pytorch_model.safetensors''',
     ]
     self.assertTrue(is_safetensors_compatible(lowerCAmelCase   )   )
    def 						UpperCamelCase_				(  self   )       ->					int:
     SCREAMING_SNAKE_CASE__:   List[str]=      [
         '''unet/diffusion_pytorch_model.bin''',
         '''unet/diffusion_pytorch_model.safetensors''',
     ]
     self.assertTrue(is_safetensors_compatible(lowerCAmelCase   )   )
    def 						UpperCamelCase_				(  self   )       ->					Tuple:
     SCREAMING_SNAKE_CASE__:   Optional[int]=      [
         '''safety_checker/pytorch_model.bin''',
         '''safety_checker/model.safetensors''',
         '''vae/diffusion_pytorch_model.bin''',
         '''vae/diffusion_pytorch_model.safetensors''',
         '''text_encoder/pytorch_model.bin''',
         '''text_encoder/model.safetensors''',
         '''unet/diffusion_pytorch_model.bin''',
         # Removed: 'unet/diffusion_pytorch_model.safetensors',
     ]
     self.assertFalse(is_safetensors_compatible(lowerCAmelCase   )   )
    def 						UpperCamelCase_				(  self   )       ->					Dict:
     SCREAMING_SNAKE_CASE__:   str=      [
         '''text_encoder/pytorch_model.bin''',
         '''text_encoder/model.safetensors''',
     ]
     self.assertTrue(is_safetensors_compatible(lowerCAmelCase   )   )
    def 						UpperCamelCase_				(  self   )       ->					str:
     SCREAMING_SNAKE_CASE__:   List[str]=      [
         '''safety_checker/pytorch_model.bin''',
         '''safety_checker/model.safetensors''',
         '''vae/diffusion_pytorch_model.bin''',
         '''vae/diffusion_pytorch_model.safetensors''',
         '''text_encoder/pytorch_model.bin''',
         # Removed: 'text_encoder/model.safetensors',
         '''unet/diffusion_pytorch_model.bin''',
         '''unet/diffusion_pytorch_model.safetensors''',
     ]
     self.assertFalse(is_safetensors_compatible(lowerCAmelCase   )   )
    def 						UpperCamelCase_				(  self   )       ->					List[str]:
     SCREAMING_SNAKE_CASE__:   Optional[int]=      [
         '''safety_checker/pytorch_model.fp16.bin''',
         '''safety_checker/model.fp16.safetensors''',
         '''vae/diffusion_pytorch_model.fp16.bin''',
         '''vae/diffusion_pytorch_model.fp16.safetensors''',
         '''text_encoder/pytorch_model.fp16.bin''',
         '''text_encoder/model.fp16.safetensors''',
         '''unet/diffusion_pytorch_model.fp16.bin''',
         '''unet/diffusion_pytorch_model.fp16.safetensors''',
     ]
     SCREAMING_SNAKE_CASE__:   Any=      '''fp16'''
     self.assertTrue(is_safetensors_compatible(lowerCAmelCase , variant=lowerCAmelCase   )   )
    def 						UpperCamelCase_				(  self   )       ->					Any:
     SCREAMING_SNAKE_CASE__:   List[str]=      [
         '''unet/diffusion_pytorch_model.fp16.bin''',
         '''unet/diffusion_pytorch_model.fp16.safetensors''',
     ]
     SCREAMING_SNAKE_CASE__:   Dict=      '''fp16'''
     self.assertTrue(is_safetensors_compatible(lowerCAmelCase , variant=lowerCAmelCase   )   )
    def 						UpperCamelCase_				(  self   )       ->					Dict:
     # pass variant but use the non-variant filenames
     SCREAMING_SNAKE_CASE__:   List[str]=      [
         '''unet/diffusion_pytorch_model.bin''',
         '''unet/diffusion_pytorch_model.safetensors''',
     ]
     SCREAMING_SNAKE_CASE__:   Union[str, Any]=      '''fp16'''
     self.assertTrue(is_safetensors_compatible(lowerCAmelCase , variant=lowerCAmelCase   )   )
    def 						UpperCamelCase_				(  self   )       ->					List[Any]:
     SCREAMING_SNAKE_CASE__:   Dict=      [
         '''safety_checker/pytorch_model.fp16.bin''',
         '''safety_checker/model.fp16.safetensors''',
         '''vae/diffusion_pytorch_model.fp16.bin''',
         '''vae/diffusion_pytorch_model.fp16.safetensors''',
         '''text_encoder/pytorch_model.fp16.bin''',
         '''text_encoder/model.fp16.safetensors''',
         '''unet/diffusion_pytorch_model.fp16.bin''',
         # Removed: 'unet/diffusion_pytorch_model.fp16.safetensors',
     ]
     SCREAMING_SNAKE_CASE__:   List[Any]=      '''fp16'''
     self.assertFalse(is_safetensors_compatible(lowerCAmelCase , variant=lowerCAmelCase   )   )
    def 						UpperCamelCase_				(  self   )       ->					List[str]:
     SCREAMING_SNAKE_CASE__:   int=      [
         '''text_encoder/pytorch_model.fp16.bin''',
         '''text_encoder/model.fp16.safetensors''',
     ]
     SCREAMING_SNAKE_CASE__:   Any=      '''fp16'''
     self.assertTrue(is_safetensors_compatible(lowerCAmelCase , variant=lowerCAmelCase   )   )
    def 						UpperCamelCase_				(  self   )       ->					Optional[Any]:
     # pass variant but use the non-variant filenames
     SCREAMING_SNAKE_CASE__:   Optional[int]=      [
         '''text_encoder/pytorch_model.bin''',
         '''text_encoder/model.safetensors''',
     ]
     SCREAMING_SNAKE_CASE__:   str=      '''fp16'''
     self.assertTrue(is_safetensors_compatible(lowerCAmelCase , variant=lowerCAmelCase   )   )



    def 						UpperCamelCase_				(  self   )       ->					Union[str, Any]:
     SCREAMING_SNAKE_CASE__:   Tuple=      [
         '''safety_checker/pytorch_model.fp16.bin''',
         '''safety_checker/model.fp16.safetensors''',
         '''vae/diffusion_pytorch_model.fp16.bin''',
         '''vae/diffusion_pytorch_model.fp16.safetensors''',
         '''text_encoder/pytorch_model.fp16.bin''',
         # 'text_encoder/model.fp16.safetensors',
         '''unet/diffusion_pytorch_model.fp16.bin''',
         '''unet/diffusion_pytorch_model.fp16.safetensors''',
     ]
     SCREAMING_SNAKE_CASE__:   str=      '''fp16'''
     self.assertFalse(is_safetensors_compatible(lowerCAmelCase , variant=lowerCAmelCase   )   )

