





'''simple docstring'''





import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


SCREAMING_SNAKE_CASE_		     = logging.get_logger(__name__)

SCREAMING_SNAKE_CASE_		     = {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

SCREAMING_SNAKE_CASE_		     = {
    """vocab_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt"""
        ),
        """google/realm-orqa-nq-openqa""": """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-nq-reader""": """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-openqa""": """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-reader""": """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt""",
    },
    """tokenizer_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-openqa""": (
            """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-reader""": (
            """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-openqa""": (
            """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-reader""": (
            """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json"""
        ),
    },
}

SCREAMING_SNAKE_CASE_		     = {
    """google/realm-cc-news-pretrained-embedder""": 5_1_2,
    """google/realm-cc-news-pretrained-encoder""": 5_1_2,
    """google/realm-cc-news-pretrained-scorer""": 5_1_2,
    """google/realm-cc-news-pretrained-openqa""": 5_1_2,
    """google/realm-orqa-nq-openqa""": 5_1_2,
    """google/realm-orqa-nq-reader""": 5_1_2,
    """google/realm-orqa-wq-openqa""": 5_1_2,
    """google/realm-orqa-wq-reader""": 5_1_2,
}

SCREAMING_SNAKE_CASE_		     = {
    """google/realm-cc-news-pretrained-embedder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-encoder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-scorer""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-reader""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-reader""": {"""do_lower_case""": True},
}

class 	UpperCamelCase__				(   lowerCAmelCase_     ):



     '''simple docstring'''

     __snake_case					:			Any				=    VOCAB_FILES_NAMES
     __snake_case					:			List[str]				=    PRETRAINED_VOCAB_FILES_MAP
     __snake_case					:			Any				=    PRETRAINED_INIT_CONFIGURATION
     __snake_case					:			Tuple				=    PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     __snake_case					:			Dict				=    RealmTokenizer






     def __init__(							self		:					Optional[Any]		,lowerCamelCase__		:					List[str]=None		,lowerCamelCase__		:					Any=None		,lowerCamelCase__		:					Dict=True		,lowerCamelCase__		:					Dict="[UNK]"		,lowerCamelCase__		:					List[Any]="[SEP]"		,lowerCamelCase__		:					List[str]="[PAD]"		,lowerCamelCase__		:					Optional[int]="[CLS]"		,lowerCamelCase__		:					List[Any]="[MASK]"		,lowerCamelCase__		:					List[str]=True		,lowerCamelCase__		:					Optional[Any]=None		,**lowerCamelCase__		:					Optional[Any]		,)  ->	List[Any]:




           '''simple docstring'''


           super().__init__(
               lowerCamelCase__		,tokenizer_file=lowerCamelCase__		,do_lower_case=lowerCamelCase__		,unk_token=lowerCamelCase__		,sep_token=lowerCamelCase__		,pad_token=lowerCamelCase__		,cls_token=lowerCamelCase__		,mask_token=lowerCamelCase__		,tokenize_chinese_chars=lowerCamelCase__		,strip_accents=lowerCamelCase__		,**lowerCamelCase__		,)

           SCREAMING_SNAKE_CASE	=		json.loads(self.backend_tokenizer.normalizer.__getstate__() )
           if (
               normalizer_state.get("""lowercase"""		,lowerCamelCase__ ) != do_lower_case
               or normalizer_state.get("""strip_accents"""		,lowerCamelCase__ ) != strip_accents
               or normalizer_state.get("""handle_chinese_chars"""		,lowerCamelCase__ ) != tokenize_chinese_chars
           ):
                 SCREAMING_SNAKE_CASE	=		getattr(lowerCamelCase__		,normalizer_state.pop("""type""" ) )
                 SCREAMING_SNAKE_CASE	=		do_lower_case
                 SCREAMING_SNAKE_CASE	=		strip_accents
                 SCREAMING_SNAKE_CASE	=		tokenize_chinese_chars
                 SCREAMING_SNAKE_CASE	=		normalizer_class(**lowerCamelCase__ )

           SCREAMING_SNAKE_CASE	=		do_lower_case






     def        SCREAMING_SNAKE_CASE__						(							self		:					Optional[int]		,lowerCamelCase__		:					Optional[int]		,**lowerCamelCase__		:					Optional[int] )  ->	str:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE	=		PaddingStrategy.MAX_LENGTH

           SCREAMING_SNAKE_CASE	=		text
           SCREAMING_SNAKE_CASE	=		kwargs.pop("""text_pair"""		,lowerCamelCase__ )
           SCREAMING_SNAKE_CASE	=		kwargs.pop("""return_tensors"""		,lowerCamelCase__ )

           SCREAMING_SNAKE_CASE	=		{
               """input_ids""": [],
               """attention_mask""": [],
               """token_type_ids""": [],
           }

           for idx, candidate_text in enumerate(lowerCamelCase__ ):
                 if batch_text_pair is not None:
                       SCREAMING_SNAKE_CASE	=		batch_text_pair[idx]
                 else:
                       SCREAMING_SNAKE_CASE	=		None

                 SCREAMING_SNAKE_CASE	=		super().__call__(lowerCamelCase__		,lowerCamelCase__		,return_tensors=lowerCamelCase__		,**lowerCamelCase__ )

                 SCREAMING_SNAKE_CASE	=		encoded_candidates.get("""input_ids""" )
                 SCREAMING_SNAKE_CASE	=		encoded_candidates.get("""attention_mask""" )
                 SCREAMING_SNAKE_CASE	=		encoded_candidates.get("""token_type_ids""" )

                 if encoded_input_ids is not None:
                       output_data["input_ids"].append(lowerCamelCase__ )
                 if encoded_attention_mask is not None:
                       output_data["attention_mask"].append(lowerCamelCase__ )
                 if encoded_token_type_ids is not None:
                       output_data["token_type_ids"].append(lowerCamelCase__ )

           SCREAMING_SNAKE_CASE	=		{key: item for key, item in output_data.items() if len(lowerCamelCase__ ) != 0}

           return BatchEncoding(lowerCamelCase__		,tensor_type=lowerCamelCase__ )






     def        SCREAMING_SNAKE_CASE__						(							self		:					Tuple		,lowerCamelCase__		:					List[str]		,lowerCamelCase__		:					List[str]=None )  ->	str:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE	=		[self.cls_token_id] + token_ids_a + [self.sep_token_id]

           if token_ids_a:
                 output += token_ids_a + [self.sep_token_id]

           return output






     def        SCREAMING_SNAKE_CASE__						(							self		:					List[str]		,lowerCamelCase__		:					List[int]		,lowerCamelCase__		:					Optional[List[int]] = None )  ->	List[int]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE	=		[self.sep_token_id]
           SCREAMING_SNAKE_CASE	=		[self.cls_token_id]
           if token_ids_a is None:
                 return len(cls + token_ids_a + sep ) * [0]
           return len(cls + token_ids_a + sep ) * [0] + len(token_ids_a + sep ) * [1]






     def        SCREAMING_SNAKE_CASE__						(							self		:					Optional[int]		,lowerCamelCase__		:					str		,lowerCamelCase__		:					Optional[str] = None )  ->	Tuple[str]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE	=		self._tokenizer.model.save(lowerCamelCase__		,name=lowerCamelCase__ )
           return tuple(lowerCamelCase__ )









import unittest

import torch

from diffusers import VQModel
from diffusers.utils import floats_tensor, torch_device
from diffusers.utils.testing_utils import enable_full_determinism

from .test_modeling_common import ModelTesterMixin, UNetTesterMixin


enable_full_determinism()

class 	UpperCamelCase__				(   lowerCAmelCase_							,   lowerCAmelCase_							,   unittest.TestCase     ):



     '''simple docstring'''

     __snake_case					:			str				=    VQModel
     __snake_case					:			Optional[Any]				=    "sample"






     @property
     def        SCREAMING_SNAKE_CASE__						(							self		:					Tuple		,lowerCamelCase__		:					Optional[Any]=(32, 32) )  ->	Optional[int]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE	=		4
           SCREAMING_SNAKE_CASE	=		3

           SCREAMING_SNAKE_CASE	=		floats_tensor((batch_size, num_channels) + sizes ).to(lowerCamelCase__ )

           return {"sample": image}






     @property
     def        SCREAMING_SNAKE_CASE__						(							self		:					Optional[Any] )  ->	List[str]:




           '''simple docstring'''


           return (3, 32, 32)






     @property
     def        SCREAMING_SNAKE_CASE__						(							self		:					Tuple )  ->	List[str]:




           '''simple docstring'''


           return (3, 32, 32)






     def        SCREAMING_SNAKE_CASE__						(							self		:					Optional[int] )  ->	Optional[int]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE	=		{
               """block_out_channels""": [32, 64],
               """in_channels""": 3,
               """out_channels""": 3,
               """down_block_types""": ["""DownEncoderBlock2D""", """DownEncoderBlock2D"""],
               """up_block_types""": ["""UpDecoderBlock2D""", """UpDecoderBlock2D"""],
               """latent_channels""": 3,
           }
           SCREAMING_SNAKE_CASE	=		self.dummy_input
           return init_dict, inputs_dict






     def        SCREAMING_SNAKE_CASE__						(							self		:					int )  ->	str:




           '''simple docstring'''


           pass






     def        SCREAMING_SNAKE_CASE__						(							self		:					Union[str, Any] )  ->	List[str]:




           '''simple docstring'''


           pass






     def        SCREAMING_SNAKE_CASE__						(							self		:					List[str] )  ->	Any:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE,      SCREAMING_SNAKE_CASE	=		VQModel.from_pretrained("""fusing/vqgan-dummy"""		,output_loading_info=lowerCamelCase__ )
           self.assertIsNotNone(lowerCamelCase__ )
           self.assertEqual(len(loading_info["""missing_keys"""] )		,0 )

           model.to(lowerCamelCase__ )
           SCREAMING_SNAKE_CASE	=		model(**self.dummy_input )

           assert image is not None, "Make sure output is not None"






     def        SCREAMING_SNAKE_CASE__						(							self		:					str )  ->	List[str]:




           '''simple docstring'''


           SCREAMING_SNAKE_CASE	=		VQModel.from_pretrained("""fusing/vqgan-dummy""" )
           model.to(lowerCamelCase__ ).eval()

           torch.manual_seed(0 )
           if torch.cuda.is_available():
                 torch.cuda.manual_seed_all(0 )

           SCREAMING_SNAKE_CASE	=		torch.randn(1		,model.config.in_channels		,model.config.sample_size		,model.config.sample_size )
           SCREAMING_SNAKE_CASE	=		image.to(lowerCamelCase__ )
           with torch.no_grad():
                 SCREAMING_SNAKE_CASE	=		model(lowerCamelCase__ ).sample

           SCREAMING_SNAKE_CASE	=		output[0, -1, -3:, -3:].flatten().cpu()
           # fmt: off
           SCREAMING_SNAKE_CASE	=		torch.tensor([-0.0153, -0.4044, -0.1880, -0.5161, -0.2418, -0.4072, -0.1612, -0.0633, -0.0143] )
           # fmt: on
           self.assertTrue(torch.allclose(lowerCamelCase__		,lowerCamelCase__		,atol=1e-3 ) )


