




'''simple docstring'''







import json
from typing import List, Optional, Tuple

from tokenizers import normalizers

from ...tokenization_utils_base import BatchEncoding
from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import PaddingStrategy, logging
from .tokenization_realm import RealmTokenizer


lowercase      :   Any      =       logging.get_logger(__name__)

lowercase      :   int      =       {"""vocab_file""": """vocab.txt""", """tokenizer_file""": """tokenizer.json"""}

lowercase      :   List[str]      =       {
    """vocab_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/vocab.txt"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/vocab.txt"""
        ),
        """google/realm-orqa-nq-openqa""": """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-nq-reader""": """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-openqa""": """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/vocab.txt""",
        """google/realm-orqa-wq-reader""": """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/vocab.txt""",
    },
    """tokenizer_file""": {
        """google/realm-cc-news-pretrained-embedder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-embedder/resolve/main/tokenizer.jsont"""
        ),
        """google/realm-cc-news-pretrained-encoder""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-encoder/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-scorer""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-scorer/resolve/main/tokenizer.json"""
        ),
        """google/realm-cc-news-pretrained-openqa""": (
            """https://huggingface.co/google/realm-cc-news-pretrained-openqa/aresolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-openqa""": (
            """https://huggingface.co/google/realm-orqa-nq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-nq-reader""": (
            """https://huggingface.co/google/realm-orqa-nq-reader/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-openqa""": (
            """https://huggingface.co/google/realm-orqa-wq-openqa/resolve/main/tokenizer.json"""
        ),
        """google/realm-orqa-wq-reader""": (
            """https://huggingface.co/google/realm-orqa-wq-reader/resolve/main/tokenizer.json"""
        ),
    },
}

lowercase      :   List[str]      =       {
    """google/realm-cc-news-pretrained-embedder""": 512,
    """google/realm-cc-news-pretrained-encoder""": 512,
    """google/realm-cc-news-pretrained-scorer""": 512,
    """google/realm-cc-news-pretrained-openqa""": 512,
    """google/realm-orqa-nq-openqa""": 512,
    """google/realm-orqa-nq-reader""": 512,
    """google/realm-orqa-wq-openqa""": 512,
    """google/realm-orqa-wq-reader""": 512,
}

lowercase      :   Optional[Any]      =       {
    """google/realm-cc-news-pretrained-embedder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-encoder""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-scorer""": {"""do_lower_case""": True},
    """google/realm-cc-news-pretrained-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-nq-reader""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-openqa""": {"""do_lower_case""": True},
    """google/realm-orqa-wq-reader""": {"""do_lower_case""": True},
}







class 					_a (a__	):





     '''simple docstring'''



     lowerCAmelCase_  :     Optional[int]          =					VOCAB_FILES_NAMES
     lowerCAmelCase_  :     int          =					PRETRAINED_VOCAB_FILES_MAP
     lowerCAmelCase_  :     str          =					PRETRAINED_INIT_CONFIGURATION
     lowerCAmelCase_  :     Any          =					PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
     lowerCAmelCase_  :     Any          =					RealmTokenizer







     def __init__(				self							,__a=None							,__a=None							,__a=True							,__a="[UNK]"							,__a="[SEP]"							,__a="[PAD]"							,__a="[CLS]"							,__a="[MASK]"							,__a=True							,__a=None							,**__a							,)      ->				Dict:
          super().__init__(
              __a							,tokenizer_file=__a							,do_lower_case=__a							,unk_token=__a							,sep_token=__a							,pad_token=__a							,cls_token=__a							,mask_token=__a							,tokenize_chinese_chars=__a							,strip_accents=__a							,**__a							,)

          snake_case			:      Any        =			json.loads(self.backend_tokenizer.normalizer.__getstate__()							)
          if (
              normalizer_state.get("""lowercase"""							,__a							) != do_lower_case
              or normalizer_state.get("""strip_accents"""							,__a							) != strip_accents
              or normalizer_state.get("""handle_chinese_chars"""							,__a							) != tokenize_chinese_chars
          ):
               snake_case			:      Optional[int]        =			getattr(__a							,normalizer_state.pop("""type"""							)							)
               snake_case			:      List[str]        =			do_lower_case
               snake_case			:      Optional[Any]        =			strip_accents
               snake_case			:      List[str]        =			tokenize_chinese_chars
               snake_case			:      Any        =			normalizer_class(**__a							)

          snake_case			:      Any        =			do_lower_case







     def 			snake_case_      (				self							,__a							,**__a							)      ->				Tuple:
          snake_case			:      List[Any]        =			PaddingStrategy.MAX_LENGTH

          snake_case			:      Tuple        =			text
          snake_case			:      str        =			kwargs.pop("""text_pair"""							,__a							)
          snake_case			:      Union[str, Any]        =			kwargs.pop("""return_tensors"""							,__a							)

          snake_case			:      Tuple        =			{
              """input_ids""": [],
              """attention_mask""": [],
              """token_type_ids""": [],
          }

          for idx, candidate_text in enumerate(__a							):
               if batch_text_pair is not None:
                    snake_case			:      List[Any]        =			batch_text_pair[idx]
               else:
                    snake_case			:      List[str]        =			None

               snake_case			:      int        =			super().__call__(__a							,__a							,return_tensors=__a							,**__a							)

               snake_case			:      str        =			encoded_candidates.get("""input_ids"""							)
               snake_case			:      Union[str, Any]        =			encoded_candidates.get("""attention_mask"""							)
               snake_case			:      Dict        =			encoded_candidates.get("""token_type_ids"""							)

               if encoded_input_ids is not None:
                    output_data["input_ids"].append(__a							)
               if encoded_attention_mask is not None:
                    output_data["attention_mask"].append(__a							)
               if encoded_token_type_ids is not None:
                    output_data["token_type_ids"].append(__a							)

          snake_case			:      int        =			{key: item for key, item in output_data.items() if len(__a							) != 0}

          return BatchEncoding(__a							,tensor_type=__a							)







     def 			snake_case_      (				self							,__a							,__a=None							)      ->				int:
          snake_case			:      Tuple        =			[self.cls_token_id] + token_ids_a + [self.sep_token_id]

          if token_ids_a:
               output += token_ids_a + [self.sep_token_id]

          return output







     def 			snake_case_      (				self							,__a							,__a = None							)      ->				List[int]:
          snake_case			:      List[Any]        =			[self.sep_token_id]
          snake_case			:      Dict        =			[self.cls_token_id]
          if token_ids_a is None:
               return len(cls + token_ids_a + sep							) * [0]
          return len(cls + token_ids_a + sep							) * [0] + len(token_ids_a + sep							) * [1]







     def 			snake_case_      (				self							,__a							,__a = None							)      ->				Tuple[str]:
          snake_case			:      Dict        =			self._tokenizer.model.save(__a							,name=__a							)
          return tuple(__a							)









'''simple docstring'''







import gc
import unittest

import numpy as np
import torch

from diffusers import DanceDiffusionPipeline, IPNDMScheduler, UNetaDModel
from diffusers.utils import slow, torch_device
from diffusers.utils.testing_utils import enable_full_determinism, require_torch_gpu, skip_mps

from ..pipeline_params import UNCONDITIONAL_AUDIO_GENERATION_BATCH_PARAMS, UNCONDITIONAL_AUDIO_GENERATION_PARAMS
from ..test_pipelines_common import PipelineTesterMixin


enable_full_determinism()







class 					_a (a__,		unittest.TestCase	):





     '''simple docstring'''



     lowerCAmelCase_  :     Dict          =					DanceDiffusionPipeline
     lowerCAmelCase_  :     int          =					UNCONDITIONAL_AUDIO_GENERATION_PARAMS
     lowerCAmelCase_  :     Optional[int]          =					PipelineTesterMixin.required_optional_params - {
         """callback""",
         """latents""",
         """callback_steps""",
         """output_type""",
         """num_images_per_prompt""",
     }
     lowerCAmelCase_  :     Tuple          =					UNCONDITIONAL_AUDIO_GENERATION_BATCH_PARAMS
     lowerCAmelCase_  :     Dict          =					False
     lowerCAmelCase_  :     Tuple          =					False







     def 			snake_case_      (				self							)      ->				List[Any]:
          torch.manual_seed(0							)
          snake_case			:      List[str]        =			UNetaDModel(
              block_out_channels=(32, 32, 64)							,extra_in_channels=16							,sample_size=512							,sample_rate=16_000							,in_channels=2							,out_channels=2							,flip_sin_to_cos=__a							,use_timestep_embedding=__a							,time_embedding_type="""fourier"""							,mid_block_type="""UNetMidBlock1D"""							,down_block_types=("""DownBlock1DNoSkip""", """DownBlock1D""", """AttnDownBlock1D""")							,up_block_types=("""AttnUpBlock1D""", """UpBlock1D""", """UpBlock1DNoSkip""")							,)
          snake_case			:      Dict        =			IPNDMScheduler()

          snake_case			:      int        =			{
              """unet""": unet,
              """scheduler""": scheduler,
          }
          return components







     def 			snake_case_      (				self							,__a							,__a=0							)      ->				Tuple:
          if str(__a							).startswith("""mps"""							):
               snake_case			:      Optional[Any]        =			torch.manual_seed(__a							)
          else:
               snake_case			:      Union[str, Any]        =			torch.Generator(device=__a							).manual_seed(__a							)
          snake_case			:      str        =			{
              """batch_size""": 1,
              """generator""": generator,
              """num_inference_steps""": 4,
          }
          return inputs







     def 			snake_case_      (				self							)      ->				Any:
          snake_case			:      Dict        =			"""cpu"""  # ensure determinism for the device-dependent torch.Generator
          snake_case			:      str        =			self.get_dummy_components()
          snake_case			:      List[str]        =			DanceDiffusionPipeline(**__a							)
          snake_case			:      Dict        =			pipe.to(__a							)
          pipe.set_progress_bar_config(disable=__a							)

          snake_case			:      Union[str, Any]        =			self.get_dummy_inputs(__a							)
          snake_case			:      str        =			pipe(**__a							)
          snake_case			:      Union[str, Any]        =			output.audios

          snake_case			:      List[str]        =			audio[0, -3:, -3:]

          assert audio.shape == (1, 2, components["unet"].sample_size)
          snake_case			:      Optional[Any]        =			np.array([-0.7_265, 1.0_000, -0.8_388, 0.1_175, 0.9_498, -1.0_000]							)
          assert np.abs(audio_slice.flatten() - expected_slice							).max() < 1E-2







     @skip_mps
     def 			snake_case_      (				self							)      ->				List[str]:
          return super().test_save_load_local()







     @skip_mps
     def 			snake_case_      (				self							)      ->				Union[str, Any]:
          return super().test_dict_tuple_outputs_equivalent(expected_max_difference=3E-3							)







     @skip_mps
     def 			snake_case_      (				self							)      ->				Dict:
          return super().test_save_load_optional_components()







     @skip_mps
     def 			snake_case_      (				self							)      ->				Any:
          return super().test_attention_slicing_forward_pass()







     def 			snake_case_      (				self							)      ->				str:
          super().test_inference_batch_single_identical(expected_max_diff=3E-3							)







@slow
@require_torch_gpu
class 					_a (unittest.TestCase	):





     '''simple docstring'''



     def 			snake_case_      (				self							)      ->				Optional[int]:
          # clean up the VRAM after each test
          super().tearDown()
          gc.collect()
          torch.cuda.empty_cache()







     def 			snake_case_      (				self							)      ->				List[Any]:
          snake_case			:      str        =			torch_device

          snake_case			:      List[str]        =			DanceDiffusionPipeline.from_pretrained("""harmonai/maestro-150k"""							)
          snake_case			:      Any        =			pipe.to(__a							)
          pipe.set_progress_bar_config(disable=__a							)

          snake_case			:      int        =			torch.manual_seed(0							)
          snake_case			:      int        =			pipe(generator=__a							,num_inference_steps=100							,audio_length_in_s=4.096							)
          snake_case			:      Dict        =			output.audios

          snake_case			:      Union[str, Any]        =			audio[0, -3:, -3:]

          assert audio.shape == (1, 2, pipe.unet.sample_size)
          snake_case			:      Tuple        =			np.array([-0.0_192, -0.0_231, -0.0_318, -0.0_059, 0.0_002, -0.0_020]							)

          assert np.abs(audio_slice.flatten() - expected_slice							).max() < 1E-2







     def 			snake_case_      (				self							)      ->				str:
          snake_case			:      str        =			torch_device

          snake_case			:      List[Any]        =			DanceDiffusionPipeline.from_pretrained("""harmonai/maestro-150k"""							,torch_dtype=torch.floataa							)
          snake_case			:      str        =			pipe.to(__a							)
          pipe.set_progress_bar_config(disable=__a							)

          snake_case			:      str        =			torch.manual_seed(0							)
          snake_case			:      List[str]        =			pipe(generator=__a							,num_inference_steps=100							,audio_length_in_s=4.096							)
          snake_case			:      Optional[int]        =			output.audios

          snake_case			:      Any        =			audio[0, -3:, -3:]

          assert audio.shape == (1, 2, pipe.unet.sample_size)
          snake_case			:      Optional[int]        =			np.array([-0.0_367, -0.0_488, -0.0_771, -0.0_525, -0.0_444, -0.0_341]							)

          assert np.abs(audio_slice.flatten() - expected_slice							).max() < 1E-2



