






"""simple docstring"""







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import logging
from .tokenization_bert import BertTokenizer


lowerCAmelCase_	:    str            =      logging.get_logger(__name__)

lowerCAmelCase_	:    Union[str, Any]            =      {"vocab_file": "vocab.txt", "tokenizer_file": "tokenizer.json"}

lowerCAmelCase_	:    List[str]            =      {
    "vocab_file": {
        "bert-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/vocab.txt",
        "bert-large-uncased": "https://huggingface.co/bert-large-uncased/resolve/main/vocab.txt",
        "bert-base-cased": "https://huggingface.co/bert-base-cased/resolve/main/vocab.txt",
        "bert-large-cased": "https://huggingface.co/bert-large-cased/resolve/main/vocab.txt",
        "bert-base-multilingual-uncased": (
            "https://huggingface.co/bert-base-multilingual-uncased/resolve/main/vocab.txt"
        ),
        "bert-base-multilingual-cased": "https://huggingface.co/bert-base-multilingual-cased/resolve/main/vocab.txt",
        "bert-base-chinese": "https://huggingface.co/bert-base-chinese/resolve/main/vocab.txt",
        "bert-base-german-cased": "https://huggingface.co/bert-base-german-cased/resolve/main/vocab.txt",
        "bert-large-uncased-whole-word-masking": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/vocab.txt"
        ),
        "bert-large-cased-whole-word-masking": (
            "https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/vocab.txt"
        ),
        "bert-large-uncased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt"
        ),
        "bert-large-cased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/vocab.txt"
        ),
        "bert-base-cased-finetuned-mrpc": (
            "https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/vocab.txt"
        ),
        "bert-base-german-dbmdz-cased": "https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/vocab.txt",
        "bert-base-german-dbmdz-uncased": (
            "https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/vocab.txt"
        ),
        "TurkuNLP/bert-base-finnish-cased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/vocab.txt"
        ),
        "TurkuNLP/bert-base-finnish-uncased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/vocab.txt"
        ),
        "wietsedv/bert-base-dutch-cased": (
            "https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/vocab.txt"
        ),
    },
    "tokenizer_file": {
        "bert-base-uncased": "https://huggingface.co/bert-base-uncased/resolve/main/tokenizer.json",
        "bert-large-uncased": "https://huggingface.co/bert-large-uncased/resolve/main/tokenizer.json",
        "bert-base-cased": "https://huggingface.co/bert-base-cased/resolve/main/tokenizer.json",
        "bert-large-cased": "https://huggingface.co/bert-large-cased/resolve/main/tokenizer.json",
        "bert-base-multilingual-uncased": (
            "https://huggingface.co/bert-base-multilingual-uncased/resolve/main/tokenizer.json"
        ),
        "bert-base-multilingual-cased": (
            "https://huggingface.co/bert-base-multilingual-cased/resolve/main/tokenizer.json"
        ),
        "bert-base-chinese": "https://huggingface.co/bert-base-chinese/resolve/main/tokenizer.json",
        "bert-base-german-cased": "https://huggingface.co/bert-base-german-cased/resolve/main/tokenizer.json",
        "bert-large-uncased-whole-word-masking": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking/resolve/main/tokenizer.json"
        ),
        "bert-large-cased-whole-word-masking": (
            "https://huggingface.co/bert-large-cased-whole-word-masking/resolve/main/tokenizer.json"
        ),
        "bert-large-uncased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-uncased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json"
        ),
        "bert-large-cased-whole-word-masking-finetuned-squad": (
            "https://huggingface.co/bert-large-cased-whole-word-masking-finetuned-squad/resolve/main/tokenizer.json"
        ),
        "bert-base-cased-finetuned-mrpc": (
            "https://huggingface.co/bert-base-cased-finetuned-mrpc/resolve/main/tokenizer.json"
        ),
        "bert-base-german-dbmdz-cased": (
            "https://huggingface.co/bert-base-german-dbmdz-cased/resolve/main/tokenizer.json"
        ),
        "bert-base-german-dbmdz-uncased": (
            "https://huggingface.co/bert-base-german-dbmdz-uncased/resolve/main/tokenizer.json"
        ),
        "TurkuNLP/bert-base-finnish-cased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-cased-v1/resolve/main/tokenizer.json"
        ),
        "TurkuNLP/bert-base-finnish-uncased-v1": (
            "https://huggingface.co/TurkuNLP/bert-base-finnish-uncased-v1/resolve/main/tokenizer.json"
        ),
        "wietsedv/bert-base-dutch-cased": (
            "https://huggingface.co/wietsedv/bert-base-dutch-cased/resolve/main/tokenizer.json"
        ),
    },
}

lowerCAmelCase_	:    str            =      {
    "bert-base-uncased": 5_1_2,
    "bert-large-uncased": 5_1_2,
    "bert-base-cased": 5_1_2,
    "bert-large-cased": 5_1_2,
    "bert-base-multilingual-uncased": 5_1_2,
    "bert-base-multilingual-cased": 5_1_2,
    "bert-base-chinese": 5_1_2,
    "bert-base-german-cased": 5_1_2,
    "bert-large-uncased-whole-word-masking": 5_1_2,
    "bert-large-cased-whole-word-masking": 5_1_2,
    "bert-large-uncased-whole-word-masking-finetuned-squad": 5_1_2,
    "bert-large-cased-whole-word-masking-finetuned-squad": 5_1_2,
    "bert-base-cased-finetuned-mrpc": 5_1_2,
    "bert-base-german-dbmdz-cased": 5_1_2,
    "bert-base-german-dbmdz-uncased": 5_1_2,
    "TurkuNLP/bert-base-finnish-cased-v1": 5_1_2,
    "TurkuNLP/bert-base-finnish-uncased-v1": 5_1_2,
    "wietsedv/bert-base-dutch-cased": 5_1_2,
}

lowerCAmelCase_	:    Union[str, Any]            =      {
    "bert-base-uncased": {"do_lower_case": True},
    "bert-large-uncased": {"do_lower_case": True},
    "bert-base-cased": {"do_lower_case": False},
    "bert-large-cased": {"do_lower_case": False},
    "bert-base-multilingual-uncased": {"do_lower_case": True},
    "bert-base-multilingual-cased": {"do_lower_case": False},
    "bert-base-chinese": {"do_lower_case": False},
    "bert-base-german-cased": {"do_lower_case": False},
    "bert-large-uncased-whole-word-masking": {"do_lower_case": True},
    "bert-large-cased-whole-word-masking": {"do_lower_case": False},
    "bert-large-uncased-whole-word-masking-finetuned-squad": {"do_lower_case": True},
    "bert-large-cased-whole-word-masking-finetuned-squad": {"do_lower_case": False},
    "bert-base-cased-finetuned-mrpc": {"do_lower_case": False},
    "bert-base-german-dbmdz-cased": {"do_lower_case": False},
    "bert-base-german-dbmdz-uncased": {"do_lower_case": True},
    "TurkuNLP/bert-base-finnish-cased-v1": {"do_lower_case": False},
    "TurkuNLP/bert-base-finnish-uncased-v1": {"do_lower_case": True},
    "wietsedv/bert-base-dutch-cased": {"do_lower_case": False},
}





class   UpperCamelCase_ (       _UpperCAmelCase  ):
  _A       :					Tuple				=	VOCAB_FILES_NAMES
  _A       :					str				=	PRETRAINED_VOCAB_FILES_MAP
  _A       :					List[Any]				=	PRETRAINED_INIT_CONFIGURATION
  _A       :					List[str]				=	PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
  _A       :					Tuple				=	BertTokenizer



  def __init__(      self ,		snake_case__=None ,		snake_case__=None ,		snake_case__=True ,		snake_case__="[UNK]" ,		snake_case__="[SEP]" ,		snake_case__="[PAD]" ,		snake_case__="[CLS]" ,		snake_case__="[MASK]" ,		snake_case__=True ,		snake_case__=None ,		**snake_case__ ,		)			->			Optional[Any]:



         """simple docstring"""





         super().__init__(
             A_ ,		tokenizer_file=A_ ,		do_lower_case=A_ ,		unk_token=A_ ,		sep_token=A_ ,		pad_token=A_ ,		cls_token=A_ ,		mask_token=A_ ,		tokenize_chinese_chars=A_ ,		strip_accents=A_ ,		**A_ ,		)

         UpperCAmelCase	    =			json.loads(self.backend_tokenizer.normalizer.__getstate__()				)
         if (
             normalizer_state.get("""lowercase""" ,		A_				) != do_lower_case
             or normalizer_state.get("""strip_accents""" ,		A_				) != strip_accents
             or normalizer_state.get("""handle_chinese_chars""" ,		A_				) != tokenize_chinese_chars
         ):
                UpperCAmelCase	    =			getattr(A_ ,		normalizer_state.pop("""type"""				)				)
                UpperCAmelCase	    =			do_lower_case
                UpperCAmelCase	    =			strip_accents
                UpperCAmelCase	    =			tokenize_chinese_chars
                UpperCAmelCase	    =			normalizer_class(**A_				)

         UpperCAmelCase	    =			do_lower_case



  def 		UpperCamelCase_				(      self ,		snake_case__ ,		snake_case__=None				)			->			Union[str, Any]:



         """simple docstring"""





         UpperCAmelCase	    =			[self.cls_token_id] + token_ids_a + [self.sep_token_id]

         if token_ids_a:
                output += token_ids_a + [self.sep_token_id]

         return output



  def 		UpperCamelCase_				(      self ,		snake_case__ ,		snake_case__ = None				)			->			List[int]:



         """simple docstring"""





         UpperCAmelCase	    =			[self.sep_token_id]
         UpperCAmelCase	    =			[self.cls_token_id]
         if token_ids_a is None:
                return len(cls + token_ids_a + sep				) * [0]
         return len(cls + token_ids_a + sep				) * [0] + len(token_ids_a + sep				) * [1]



  def 		UpperCamelCase_				(      self ,		snake_case__ ,		snake_case__ = None				)			->			Tuple[str]:



         """simple docstring"""





         UpperCAmelCase	    =			self._tokenizer.model.save(A_ ,		name=A_				)
         return tuple(A_				)








"""simple docstring"""







import tempfile

import numpy as np
import torch
from transformers import AutoTokenizer, TaEncoderModel

from diffusers import DDPMScheduler, UNetaDConditionModel
from diffusers.models.attention_processor import AttnAddedKVProcessor
from diffusers.pipelines.deepfloyd_if import IFWatermarker
from diffusers.utils.testing_utils import torch_device

from ..test_pipelines_common import to_np





class   UpperCamelCase_ :



  def 		UpperCamelCase_				(      self				)			->			Any:



         """simple docstring"""





         torch.manual_seed(0				)
         UpperCAmelCase	    =			TaEncoderModel.from_pretrained("""hf-internal-testing/tiny-random-t5"""				)

         torch.manual_seed(0				)
         UpperCAmelCase	    =			AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-t5"""				)

         torch.manual_seed(0				)
         UpperCAmelCase	    =			UNetaDConditionModel(
             sample_size=32 ,		layers_per_block=1 ,		block_out_channels=[32, 64] ,		down_block_types=[
                 """ResnetDownsampleBlock2D""",
                 """SimpleCrossAttnDownBlock2D""",
             ] ,		mid_block_type="""UNetMidBlock2DSimpleCrossAttn""" ,		up_block_types=["""SimpleCrossAttnUpBlock2D""", """ResnetUpsampleBlock2D"""] ,		in_channels=3 ,		out_channels=6 ,		cross_attention_dim=32 ,		encoder_hid_dim=32 ,		attention_head_dim=8 ,		addition_embed_type="""text""" ,		addition_embed_type_num_heads=2 ,		cross_attention_norm="""group_norm""" ,		resnet_time_scale_shift="""scale_shift""" ,		act_fn="""gelu""" ,		)
         unet.set_attn_processor(AttnAddedKVProcessor()				)  # For reproducibility tests

         torch.manual_seed(0				)
         UpperCAmelCase	    =			DDPMScheduler(
             num_train_timesteps=10_00 ,		beta_schedule="""squaredcos_cap_v2""" ,		beta_start=0.0_001 ,		beta_end=0.02 ,		thresholding=snake_case__ ,		dynamic_thresholding_ratio=0.95 ,		sample_max_value=1.0 ,		prediction_type="""epsilon""" ,		variance_type="""learned_range""" ,		)

         torch.manual_seed(0				)
         UpperCAmelCase	    =			IFWatermarker()

         return {
             "text_encoder": text_encoder,
             "tokenizer": tokenizer,
             "unet": unet,
             "scheduler": scheduler,
             "watermarker": watermarker,
             "safety_checker": None,
             "feature_extractor": None,
         }



  def 		UpperCamelCase_				(      self				)			->			int:



         """simple docstring"""





         torch.manual_seed(0				)
         UpperCAmelCase	    =			TaEncoderModel.from_pretrained("""hf-internal-testing/tiny-random-t5"""				)

         torch.manual_seed(0				)
         UpperCAmelCase	    =			AutoTokenizer.from_pretrained("""hf-internal-testing/tiny-random-t5"""				)

         torch.manual_seed(0				)
         UpperCAmelCase	    =			UNetaDConditionModel(
             sample_size=32 ,		layers_per_block=[1, 2] ,		block_out_channels=[32, 64] ,		down_block_types=[
                 """ResnetDownsampleBlock2D""",
                 """SimpleCrossAttnDownBlock2D""",
             ] ,		mid_block_type="""UNetMidBlock2DSimpleCrossAttn""" ,		up_block_types=["""SimpleCrossAttnUpBlock2D""", """ResnetUpsampleBlock2D"""] ,		in_channels=6 ,		out_channels=6 ,		cross_attention_dim=32 ,		encoder_hid_dim=32 ,		attention_head_dim=8 ,		addition_embed_type="""text""" ,		addition_embed_type_num_heads=2 ,		cross_attention_norm="""group_norm""" ,		resnet_time_scale_shift="""scale_shift""" ,		act_fn="""gelu""" ,		class_embed_type="""timestep""" ,		mid_block_scale_factor=1.414 ,		time_embedding_act_fn="""gelu""" ,		time_embedding_dim=32 ,		)
         unet.set_attn_processor(AttnAddedKVProcessor()				)  # For reproducibility tests

         torch.manual_seed(0				)
         UpperCAmelCase	    =			DDPMScheduler(
             num_train_timesteps=10_00 ,		beta_schedule="""squaredcos_cap_v2""" ,		beta_start=0.0_001 ,		beta_end=0.02 ,		thresholding=snake_case__ ,		dynamic_thresholding_ratio=0.95 ,		sample_max_value=1.0 ,		prediction_type="""epsilon""" ,		variance_type="""learned_range""" ,		)

         torch.manual_seed(0				)
         UpperCAmelCase	    =			DDPMScheduler(
             num_train_timesteps=10_00 ,		beta_schedule="""squaredcos_cap_v2""" ,		beta_start=0.0_001 ,		beta_end=0.02 ,		)

         torch.manual_seed(0				)
         UpperCAmelCase	    =			IFWatermarker()

         return {
             "text_encoder": text_encoder,
             "tokenizer": tokenizer,
             "unet": unet,
             "scheduler": scheduler,
             "image_noising_scheduler": image_noising_scheduler,
             "watermarker": watermarker,
             "safety_checker": None,
             "feature_extractor": None,
         }



  def 		UpperCamelCase_				(      self				)			->			Tuple:



         """simple docstring"""





         UpperCAmelCase	    =			self.get_dummy_components()
         UpperCAmelCase	    =			self.pipeline_class(**snake_case__				)
         pipe.to(snake_case__				)
         pipe.set_progress_bar_config(disable=snake_case__				)

         UpperCAmelCase	    =			self.get_dummy_inputs(snake_case__				)

         UpperCAmelCase	    =			inputs["""prompt"""]
         UpperCAmelCase	    =			inputs["""generator"""]
         UpperCAmelCase	    =			inputs["""num_inference_steps"""]
         UpperCAmelCase	    =			inputs["""output_type"""]

         if "image" in inputs:
                UpperCAmelCase	    =			inputs["""image"""]
         else:
                UpperCAmelCase	    =			None

         if "mask_image" in inputs:
                UpperCAmelCase	    =			inputs["""mask_image"""]
         else:
                UpperCAmelCase	    =			None

         if "original_image" in inputs:
                UpperCAmelCase	    =			inputs["""original_image"""]
         else:
                UpperCAmelCase	    =			None

         UpperCAmelCase    ,       UpperCAmelCase	    =			pipe.encode_prompt(snake_case__				)

         # inputs with prompt converted to embeddings
         UpperCAmelCase	    =			{
             """prompt_embeds""": prompt_embeds,
             """negative_prompt_embeds""": negative_prompt_embeds,
             """generator""": generator,
             """num_inference_steps""": num_inference_steps,
             """output_type""": output_type,
         }

         if image is not None:
                UpperCAmelCase	    =			image

         if mask_image is not None:
                UpperCAmelCase	    =			mask_image

         if original_image is not None:
                UpperCAmelCase	    =			original_image

         # set all optional components to None
         for optional_component in pipe._optional_components:
                setattr(snake_case__ ,		snake_case__ ,		snake_case__				)

         UpperCAmelCase	    =			pipe(**snake_case__				)[0]

         with tempfile.TemporaryDirectory() as tmpdir:
                pipe.save_pretrained(snake_case__				)
                UpperCAmelCase	    =			self.pipeline_class.from_pretrained(snake_case__				)
                pipe_loaded.to(snake_case__				)
                pipe_loaded.set_progress_bar_config(disable=snake_case__				)

         pipe_loaded.unet.set_attn_processor(AttnAddedKVProcessor()				)  # For reproducibility tests

         for optional_component in pipe._optional_components:
                self.assertTrue(
                    getattr(snake_case__ ,		snake_case__				) is None ,		f'''`{optional_component}` did not stay set to None after loading.''' ,		)

         UpperCAmelCase	    =			self.get_dummy_inputs(snake_case__				)

         UpperCAmelCase	    =			inputs["""generator"""]
         UpperCAmelCase	    =			inputs["""num_inference_steps"""]
         UpperCAmelCase	    =			inputs["""output_type"""]

         # inputs with prompt converted to embeddings
         UpperCAmelCase	    =			{
             """prompt_embeds""": prompt_embeds,
             """negative_prompt_embeds""": negative_prompt_embeds,
             """generator""": generator,
             """num_inference_steps""": num_inference_steps,
             """output_type""": output_type,
         }

         if image is not None:
                UpperCAmelCase	    =			image

         if mask_image is not None:
                UpperCAmelCase	    =			mask_image

         if original_image is not None:
                UpperCAmelCase	    =			original_image

         UpperCAmelCase	    =			pipe_loaded(**snake_case__				)[0]

         UpperCAmelCase	    =			np.abs(to_np(snake_case__				) - to_np(snake_case__				)				).max()
         self.assertLess(snake_case__ ,		1e-4				)



  def 		UpperCamelCase_				(      self				)			->			int:



         """simple docstring"""





         UpperCAmelCase	    =			self.get_dummy_components()
         UpperCAmelCase	    =			self.pipeline_class(**snake_case__				)
         pipe.to(snake_case__				)
         pipe.set_progress_bar_config(disable=snake_case__				)

         UpperCAmelCase	    =			self.get_dummy_inputs(snake_case__				)
         UpperCAmelCase	    =			pipe(**snake_case__				)[0]

         with tempfile.TemporaryDirectory() as tmpdir:
                pipe.save_pretrained(snake_case__				)
                UpperCAmelCase	    =			self.pipeline_class.from_pretrained(snake_case__				)
                pipe_loaded.to(snake_case__				)
                pipe_loaded.set_progress_bar_config(disable=snake_case__				)

         pipe_loaded.unet.set_attn_processor(AttnAddedKVProcessor()				)  # For reproducibility tests

         UpperCAmelCase	    =			self.get_dummy_inputs(snake_case__				)
         UpperCAmelCase	    =			pipe_loaded(**snake_case__				)[0]

         UpperCAmelCase	    =			np.abs(to_np(snake_case__				) - to_np(snake_case__				)				).max()
         self.assertLess(snake_case__ ,		1e-4				)
