'''simple docstring'''
import json
import os
import re
import shutil
import tempfile
import unittest
from typing import Tuple

from transformers import AddedToken, BatchEncoding, ByTaTokenizer
from transformers.utils import cached_property, is_tf_available, is_torch_available

from ...test_tokenization_common import TokenizerTesterMixin


if is_torch_available():
  a=  '''pt'''
elif is_tf_available():
  a=  '''tf'''
else:
  a=  '''jax'''






class 			__lowercase		(						_lowerCamelCase			,							unittest.TestCase						):







     """simple docstring"""






     SCREAMING_SNAKE_CASE__		   =						ByTaTokenizer
     SCREAMING_SNAKE_CASE__		   =						False







     def lowerCAmelCase     (		self    ):
         super().setUp()
         __UpperCamelCase	:	List[str]		    =  ByTaTokenizer()
         tokenizer.save_pretrained(self.tmpdirname    )







     @cached_property
     def lowerCAmelCase     (		self    ):
         return ByTaTokenizer.from_pretrained('google/byt5-small'    )







     def lowerCAmelCase     (		self     ,			**_lowerCamelCase    ):
         return self.tokenizer_class.from_pretrained(self.tmpdirname     ,			**_lowerCamelCase    )







     def lowerCAmelCase     (		self     ,			_lowerCamelCase     ,			_lowerCamelCase=False     ,			_lowerCamelCase=2_0     ,			_lowerCamelCase=5    ):
         # XXX The default common tokenizer tests assume that every ID is decodable on its own.
         # This assumption is invalid for ByT5 because single bytes might not be
         # valid utf-8 (byte 128 for instance).
         # Here we're overriding the smallest possible method to provide
         # a clean sequence without making the same assumption.

         __UpperCamelCase	:	List[str]		    =  []
         for i in range(len(_lowerCamelCase    )    ):
             try:
                 __UpperCamelCase	:	List[str]		    =  tokenizer.decode([i]     ,			clean_up_tokenization_spaces=_lowerCamelCase    )
             except UnicodeDecodeError:
                 pass
             toks.append((i, tok)    )

         __UpperCamelCase	:	Dict		    =  list(filter(lambda _lowerCamelCase    : re.match(R'^[ a-zA-Z]+$'     ,			t[1]    )     ,			_lowerCamelCase    )    )
         __UpperCamelCase	:	Optional[int]		    =  list(filter(lambda _lowerCamelCase    : [t[0]] == tokenizer.encode(t[1]     ,			add_special_tokens=_lowerCamelCase    )     ,			_lowerCamelCase    )    )
         if max_length is not None and len(_lowerCamelCase    ) > max_length:
             __UpperCamelCase	:	Union[str, Any]		    =  toks[:max_length]
         if min_length is not None and len(_lowerCamelCase    ) < min_length and len(_lowerCamelCase    ) > 0:
             while len(_lowerCamelCase    ) < min_length:
                 __UpperCamelCase	:	Tuple		    =  toks + toks
        # toks_str = [t[1] for t in toks]
         __UpperCamelCase	:	Dict		    =  [t[0] for t in toks]

         # Ensure consistency
         __UpperCamelCase	:	int		    =  tokenizer.decode(_lowerCamelCase     ,			clean_up_tokenization_spaces=_lowerCamelCase    )
         if " " not in output_txt and len(_lowerCamelCase    ) > 1:
             __UpperCamelCase	:	Union[str, Any]		    =  (
                 tokenizer.decode([toks_ids[0]]     ,			clean_up_tokenization_spaces=_lowerCamelCase    )
                 + ' '
                 + tokenizer.decode(toks_ids[1:]     ,			clean_up_tokenization_spaces=_lowerCamelCase    )
             )
         if with_prefix_space:
             __UpperCamelCase	:	Any		    =  ' ' + output_txt
         __UpperCamelCase	:	Any		    =  tokenizer.encode(_lowerCamelCase     ,			add_special_tokens=_lowerCamelCase    )
         return output_txt, output_ids







     def lowerCAmelCase     (		self    ):
         __UpperCamelCase	:	Union[str, Any]		    =  self.ta_base_tokenizer
         __UpperCamelCase	:	Union[str, Any]		    =  tokenizer(['hi</s>', 'I went to the gym</s>', '</s>']    )
         __UpperCamelCase	:	str		    =  tokenizer(['hi', 'I went to the gym', '']    )
         self.assertListEqual(batch_with_eos_added['input_ids']     ,			batch_without_eos_added['input_ids']    )







     def lowerCAmelCase     (		self    ):
         __UpperCamelCase	:	Dict		    =  self.ta_base_tokenizer
         __UpperCamelCase	:	Optional[int]		    =  'Unicode €.'
         __UpperCamelCase	:	str		    =  tokenizer(_lowerCamelCase    )
         __UpperCamelCase	:	Any		    =  [8_8, 1_1_3, 1_0_8, 1_0_2, 1_1_4, 1_0_3, 1_0_4, 3_5, 2_2_9, 1_3_3, 1_7_5, 4_9, 1]
         self.assertEqual(encoded['input_ids']     ,			_lowerCamelCase    )

         # decoding
         __UpperCamelCase	:	Dict		    =  tokenizer.decode(_lowerCamelCase    )
         self.assertEqual(_lowerCamelCase     ,			'Unicode €.</s>'    )

         __UpperCamelCase	:	Optional[int]		    =  tokenizer('e è é ê ë'    )
         __UpperCamelCase	:	Optional[Any]		    =  [1_0_4, 3_5, 1_9_8, 1_7_1, 3_5, 1_9_8, 1_7_2, 3_5, 1_9_8, 1_7_3, 3_5, 1_9_8, 1_7_4, 1]
         self.assertEqual(encoded['input_ids']     ,			_lowerCamelCase    )
         # decoding
         __UpperCamelCase	:	Optional[int]		    =  tokenizer.decode(_lowerCamelCase    )
         self.assertEqual(_lowerCamelCase     ,			'e è é ê ë</s>'    )

         # encode/decode, but with `encode` instead of `__call__`
         self.assertEqual(tokenizer.decode(tokenizer.encode('e è é ê ë'    )    )     ,			'e è é ê ë</s>'    )







     def lowerCAmelCase     (		self    ):
         __UpperCamelCase	:	Optional[int]		    =  self.ta_base_tokenizer
         __UpperCamelCase	:	Union[str, Any]		    =  ['A long paragraph for summarization.', 'Another paragraph for summarization.']
         # fmt: off
         __UpperCamelCase	:	str		    =  [6_8, 3_5, 1_1_1, 1_1_4, 1_1_3, 1_0_6, 3_5, 1_1_5, 1_0_0, 1_1_7, 1_0_0, 1_0_6, 1_1_7, 1_0_0, 1_1_5, 1_0_7, 3_5, 1_0_5, 1_1_4, 1_1_7, 3_5, 1_1_8, 1_2_0, 1_1_2, 1_1_2, 1_0_0, 1_1_7, 1_0_8, 1_2_5, 1_0_0, 1_1_9, 1_0_8, 1_1_4, 1_1_3, 4_9, 1, 0]
         # fmt: on
         __UpperCamelCase	:	Union[str, Any]		    =  tokenizer(_lowerCamelCase     ,			padding=_lowerCamelCase     ,			return_tensors=_lowerCamelCase    )
         self.assertIsInstance(_lowerCamelCase     ,			_lowerCamelCase    )

         if FRAMEWORK != "jax":
             __UpperCamelCase	:	List[str]		    =  list(batch.input_ids.numpy()[0]    )
         else:
             __UpperCamelCase	:	Optional[int]		    =  list(batch.input_ids.tolist()[0]    )

         self.assertListEqual(_lowerCamelCase     ,			_lowerCamelCase    )

         self.assertEqual((2, 3_7)     ,			batch.input_ids.shape    )
         self.assertEqual((2, 3_7)     ,			batch.attention_mask.shape    )







     def lowerCAmelCase     (		self    ):
         __UpperCamelCase	:	Any		    =  self.ta_base_tokenizer
         __UpperCamelCase	:	Optional[int]		    =  ['A long paragraph for summarization.', 'Another paragraph for summarization.']
         __UpperCamelCase	:	Optional[int]		    =  tokenizer(_lowerCamelCase     ,			padding=_lowerCamelCase     ,			return_tensors=_lowerCamelCase    )
         # check if input_ids are returned and no decoder_input_ids
         self.assertIn('input_ids'     ,			_lowerCamelCase    )
         self.assertIn('attention_mask'     ,			_lowerCamelCase    )
         self.assertNotIn('decoder_input_ids'     ,			_lowerCamelCase    )
         self.assertNotIn('decoder_attention_mask'     ,			_lowerCamelCase    )







     def lowerCAmelCase     (		self    ):
         __UpperCamelCase	:	Any		    =  self.ta_base_tokenizer
         __UpperCamelCase	:	Any		    =  [
             'Summary of the text.',
             'Another summary.',
         ]
         __UpperCamelCase	:	List[Any]		    =  tokenizer(
             text_target=_lowerCamelCase     ,			max_length=3_2     ,			padding='max_length'     ,			truncation=_lowerCamelCase     ,			return_tensors=_lowerCamelCase    )
         self.assertEqual(3_2     ,			targets['input_ids'].shape[1]    )







     def lowerCAmelCase     (		self    ):
         __UpperCamelCase	:	Dict		    =  self.ta_base_tokenizer
         __UpperCamelCase	:	Optional[int]		    =  ['A long paragraph for summarization. </s>']
         __UpperCamelCase	:	int		    =  ['Summary of the text. </s>']
         # fmt: off
         __UpperCamelCase	:	List[Any]		    =  [6_8, 3_5, 1_1_1, 1_1_4, 1_1_3, 1_0_6, 3_5, 1_1_5, 1_0_0, 1_1_7, 1_0_0, 1_0_6, 1_1_7, 1_0_0, 1_1_5, 1_0_7, 3_5, 1_0_5, 1_1_4, 1_1_7, 3_5, 1_1_8, 1_2_0, 1_1_2, 1_1_2, 1_0_0, 1_1_7, 1_0_8, 1_2_5, 1_0_0, 1_1_9, 1_0_8, 1_1_4, 1_1_3, 4_9, 3_5, 1]
         __UpperCamelCase	:	str		    =  [8_6, 1_2_0, 1_1_2, 1_1_2, 1_0_0, 1_1_7, 1_2_4, 3_5, 1_1_4, 1_0_5, 3_5, 1_1_9, 1_0_7, 1_0_4, 3_5, 1_1_9, 1_0_4, 1_2_3, 1_1_9, 4_9, 3_5, 1]
         # fmt: on

         __UpperCamelCase	:	Dict		    =  tokenizer(_lowerCamelCase     ,			text_target=_lowerCamelCase    )

         self.assertEqual(_lowerCamelCase     ,			batch['input_ids'][0]    )
         self.assertEqual(_lowerCamelCase     ,			batch['labels'][0]    )







     def lowerCAmelCase     (		self    ):
         # safety check on max_len default value so we are sure the test works
         __UpperCamelCase	:	Any		    =  self.get_tokenizers()
         for tokenizer in tokenizers:
             with self.subTest(f"""{tokenizer.__class__.__name__}"""    ):
                 self.assertNotEqual(tokenizer.model_max_length     ,			4_2    )

        # Now let's start the test
         __UpperCamelCase	:	int		    =  self.get_tokenizers()
         for tokenizer in tokenizers:
             with self.subTest(f"""{tokenizer.__class__.__name__}"""    ):
                 # Isolate this from the other tests because we save additional tokens/etc
                 __UpperCamelCase	:	Optional[int]		    =  tempfile.mkdtemp()

                 __UpperCamelCase	:	Dict		    =  ' He is very happy, UNwant\u00E9d,running'
                 __UpperCamelCase	:	List[Any]		    =  tokenizer.encode(_lowerCamelCase     ,			add_special_tokens=_lowerCamelCase    )
                 tokenizer.save_pretrained(_lowerCamelCase    )

                 __UpperCamelCase	:	Dict		    =  tokenizer.__class__.from_pretrained(_lowerCamelCase    )
                 __UpperCamelCase	:	Union[str, Any]		    =  after_tokenizer.encode(_lowerCamelCase     ,			add_special_tokens=_lowerCamelCase    )
                 self.assertListEqual(_lowerCamelCase     ,			_lowerCamelCase    )

                 shutil.rmtree(_lowerCamelCase    )

         __UpperCamelCase	:	Union[str, Any]		    =  self.get_tokenizers(model_max_length=4_2    )
         for tokenizer in tokenizers:
             with self.subTest(f"""{tokenizer.__class__.__name__}"""    ):
                 # Isolate this from the other tests because we save additional tokens/etc
                 __UpperCamelCase	:	Dict		    =  tempfile.mkdtemp()

                 __UpperCamelCase	:	List[str]		    =  ' He is very happy, UNwant\u00E9d,running'
                 tokenizer.add_tokens(['bim', 'bambam']    )
                 __UpperCamelCase	:	Optional[Any]		    =  tokenizer.additional_special_tokens
                 additional_special_tokens.append('new_additional_special_token'    )
                 tokenizer.add_special_tokens({'additional_special_tokens': additional_special_tokens}    )
                 __UpperCamelCase	:	Dict		    =  tokenizer.encode(_lowerCamelCase     ,			add_special_tokens=_lowerCamelCase    )
                 tokenizer.save_pretrained(_lowerCamelCase    )

                 __UpperCamelCase	:	List[Any]		    =  tokenizer.__class__.from_pretrained(_lowerCamelCase    )
                 __UpperCamelCase	:	Union[str, Any]		    =  after_tokenizer.encode(_lowerCamelCase     ,			add_special_tokens=_lowerCamelCase    )
                 self.assertListEqual(_lowerCamelCase     ,			_lowerCamelCase    )
                 self.assertIn('new_additional_special_token'     ,			after_tokenizer.additional_special_tokens    )
                 self.assertEqual(after_tokenizer.model_max_length     ,			4_2    )

                 __UpperCamelCase	:	Any		    =  tokenizer.__class__.from_pretrained(_lowerCamelCase     ,			model_max_length=4_3    )
                 self.assertEqual(tokenizer.model_max_length     ,			4_3    )

                 shutil.rmtree(_lowerCamelCase    )







     def lowerCAmelCase     (		self    ):
         __UpperCamelCase	:	List[Any]		    =  []
         if self.test_slow_tokenizer:
             tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())    )

         if self.test_rust_tokenizer:
             tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())    )

         for tokenizer_class, tokenizer_utils in tokenizer_list:
             with tempfile.TemporaryDirectory() as tmp_dir:
                 tokenizer_utils.save_pretrained(_lowerCamelCase    )

                 with open(os.path.join(_lowerCamelCase     ,			'special_tokens_map.json'    )     ,			encoding='utf-8'    ) as json_file:
                     __UpperCamelCase	:	List[str]		    =  json.load(_lowerCamelCase    )

                 with open(os.path.join(_lowerCamelCase     ,			'tokenizer_config.json'    )     ,			encoding='utf-8'    ) as json_file:
                     __UpperCamelCase	:	int		    =  json.load(_lowerCamelCase    )

                 __UpperCamelCase	:	str		    =  [f"""<extra_id_{i}>""" for i in range(1_2_5    )]

                 __UpperCamelCase	:	str		    =  added_tokens_extra_ids + [
                     'an_additional_special_token'
                 ]
                 __UpperCamelCase	:	Dict		    =  added_tokens_extra_ids + [
                     'an_additional_special_token'
                 ]

                 with open(os.path.join(_lowerCamelCase     ,			'special_tokens_map.json'    )     ,			'w'     ,			encoding='utf-8'    ) as outfile:
                     json.dump(_lowerCamelCase     ,			_lowerCamelCase    )
                 with open(os.path.join(_lowerCamelCase     ,			'tokenizer_config.json'    )     ,			'w'     ,			encoding='utf-8'    ) as outfile:
                     json.dump(_lowerCamelCase     ,			_lowerCamelCase    )

                 # the following checks allow us to verify that our test works as expected, i.e. that the tokenizer takes
                 # into account the new value of additional_special_tokens given in the "tokenizer_config.json" and
                 # "special_tokens_map.json" files
                 __UpperCamelCase	:	Optional[Any]		    =  tokenizer_class.from_pretrained(
                     _lowerCamelCase     ,			)
                 self.assertIn(
                     'an_additional_special_token'     ,			tokenizer_without_change_in_init.additional_special_tokens    )
                 # self.assertIn("an_additional_special_token",tokenizer_without_change_in_init.get_vocab()) # ByT5Tokenization no vocab
                 self.assertEqual(
                     ['an_additional_special_token']     ,			tokenizer_without_change_in_init.convert_ids_to_tokens(
                         tokenizer_without_change_in_init.convert_tokens_to_ids(['an_additional_special_token']    )    )     ,			)

                 # Now we test that we can change the value of additional_special_tokens in the from_pretrained
                 __UpperCamelCase	:	int		    =  added_tokens_extra_ids + [AddedToken('a_new_additional_special_token'     ,			lstrip=_lowerCamelCase    )]
                 __UpperCamelCase	:	int		    =  tokenizer_class.from_pretrained(
                     _lowerCamelCase     ,			additional_special_tokens=_lowerCamelCase     ,			)

                 self.assertIn('a_new_additional_special_token'     ,			tokenizer.additional_special_tokens    )
                 self.assertEqual(
                     ['a_new_additional_special_token']     ,			tokenizer.convert_ids_to_tokens(
                         tokenizer.convert_tokens_to_ids(['a_new_additional_special_token']    )    )     ,			)







     def lowerCAmelCase     (		self    ):
         __UpperCamelCase	:	Any		    =  []
         if self.test_slow_tokenizer:
             tokenizer_list.append((self.tokenizer_class, self.get_tokenizer())    )

         if self.test_rust_tokenizer:
             tokenizer_list.append((self.rust_tokenizer_class, self.get_rust_tokenizer())    )

         for tokenizer_class, tokenizer_utils in tokenizer_list:
             with tempfile.TemporaryDirectory() as tmp_dir:
                 tokenizer_utils.save_pretrained(_lowerCamelCase    )

                 __UpperCamelCase	:	str		    =  tokenizer_class.from_pretrained(_lowerCamelCase    )

                 self.assertTrue(tokenizer.decode([2_5_5]    ) == ''    )







     def lowerCAmelCase     (		self    ):
         pass







     def lowerCAmelCase     (		self    ):
         pass







     def lowerCAmelCase     (		self    ):
         pass







     def lowerCAmelCase     (		self    ):
         pass







     def lowerCAmelCase     (		self    ):
         # The default common tokenizer tests uses invalid tokens for ByT5 that can only accept one-character strings
         # and special added tokens as tokens
         __UpperCamelCase	:	Union[str, Any]		    =  self.get_tokenizers(fast=_lowerCamelCase     ,			do_lower_case=_lowerCamelCase    )
         for tokenizer in tokenizers:
             with self.subTest(f"""{tokenizer.__class__.__name__}"""    ):
                 __UpperCamelCase	:	Tuple		    =  ['t', 'h', 'i', 's', ' ', 'i', 's', ' ', 'a', ' ', 't', 'e', 'x', 't', '</s>']
                 __UpperCamelCase	:	Optional[Any]		    =  tokenizer.convert_tokens_to_string(_lowerCamelCase    )

                 self.assertIsInstance(_lowerCamelCase     ,			_lowerCamelCase    )







     def lowerCAmelCase     (		self    ):
         __UpperCamelCase	:	Optional[Any]		    =  self.get_tokenizers()
         for tokenizer in tokenizers:
             with self.subTest(f"""{tokenizer.__class__.__name__}"""    ):
                 __UpperCamelCase	:	Any		    =  [
                     'bos_token',
                     'eos_token',
                     'unk_token',
                     'sep_token',
                     'pad_token',
                     'cls_token',
                     'mask_token',
                 ]

                 __UpperCamelCase	:	Union[str, Any]		    =  0
                 __UpperCamelCase	:	Optional[Any]		    =  tokenizer.convert_ids_to_tokens(
                     _lowerCamelCase     ,			skip_special_tokens=_lowerCamelCase    )

                 for attr in attributes_list:
                     setattr(_lowerCamelCase     ,			attr + '_id'     ,			_lowerCamelCase    )
                     self.assertEqual(getattr(_lowerCamelCase     ,			_lowerCamelCase    )     ,			_lowerCamelCase    )
                     self.assertEqual(getattr(_lowerCamelCase     ,			attr + '_id'    )     ,			_lowerCamelCase    )

                     setattr(_lowerCamelCase     ,			attr + '_id'     ,			_lowerCamelCase    )
                     self.assertEqual(getattr(_lowerCamelCase     ,			_lowerCamelCase    )     ,			_lowerCamelCase    )
                     self.assertEqual(getattr(_lowerCamelCase     ,			attr + '_id'    )     ,			_lowerCamelCase    )

                 setattr(_lowerCamelCase     ,			'additional_special_tokens_ids'     ,			[]    )
                 self.assertListEqual(getattr(_lowerCamelCase     ,			'additional_special_tokens'    )     ,			[]    )
                 self.assertListEqual(getattr(_lowerCamelCase     ,			'additional_special_tokens_ids'    )     ,			[]    )

                 setattr(_lowerCamelCase     ,			'additional_special_tokens_ids'     ,			[token_id_to_test_setters]    )
                 self.assertListEqual(getattr(_lowerCamelCase     ,			'additional_special_tokens'    )     ,			[token_to_test_setters]    )
                 self.assertListEqual(getattr(_lowerCamelCase     ,			'additional_special_tokens_ids'    )     ,			[token_id_to_test_setters]    )




'''simple docstring'''
import torch
from torch import nn

from ...configuration_utils import ConfigMixin, register_to_config
from ...models import ModelMixin






class 			__lowercase		(						_lowerCamelCase			,							_lowerCamelCase						):







     """simple docstring"""






     @register_to_config
     def __init__(		self     ,			*,
         _lowerCamelCase = 4     ,			_lowerCamelCase = 7_6_8     ,			_lowerCamelCase     ,			_lowerCamelCase     ,			):
         super().__init__()

         __UpperCamelCase	:	Optional[int]		    =  nn.Parameter(torch.zeros(_lowerCamelCase    )    )

         # parameters for additional clip time embeddings
         __UpperCamelCase	:	Dict		    =  nn.Linear(_lowerCamelCase     ,			_lowerCamelCase    )
         __UpperCamelCase	:	int		    =  nn.Linear(_lowerCamelCase     ,			_lowerCamelCase    )

         # parameters for encoder hidden states
         __UpperCamelCase	:	Optional[Any]		    =  clip_extra_context_tokens
         __UpperCamelCase	:	Dict		    =  nn.Linear(
             _lowerCamelCase     ,			self.clip_extra_context_tokens * cross_attention_dim    )
         __UpperCamelCase	:	str		    =  nn.Linear(_lowerCamelCase     ,			_lowerCamelCase    )
         __UpperCamelCase	:	List[str]		    =  nn.LayerNorm(_lowerCamelCase    )







     def lowerCAmelCase     (		self     ,			*, _lowerCamelCase     ,			_lowerCamelCase     ,			_lowerCamelCase     ,			_lowerCamelCase    ):
         if do_classifier_free_guidance:
             # Add the classifier free guidance embeddings to the image embeddings
             __UpperCamelCase	:	int		    =  image_embeddings.shape[0]
             __UpperCamelCase	:	Optional[Any]		    =  self.learned_classifier_free_guidance_embeddings.unsqueeze(0    )
             __UpperCamelCase	:	List[Any]		    =  classifier_free_guidance_embeddings.expand(
                 _lowerCamelCase     ,			-1    )
             __UpperCamelCase	:	Optional[Any]		    =  torch.cat([classifier_free_guidance_embeddings, image_embeddings]     ,			dim=0    )

         # The image embeddings batch size and the text embeddings batch size are equal
         assert image_embeddings.shape[0] == prompt_embeds.shape[0]

         __UpperCamelCase	:	Optional[Any]		    =  prompt_embeds.shape[0]

         # "Specifically, we modify the architecture described in Nichol et al. (2021) by projecting and
         # adding CLIP embeddings to the existing timestep embedding, ...
         __UpperCamelCase	:	Optional[int]		    =  self.embedding_proj(_lowerCamelCase    )
         __UpperCamelCase	:	List[str]		    =  self.clip_image_embeddings_project_to_time_embeddings(_lowerCamelCase    )
         __UpperCamelCase	:	Optional[int]		    =  time_projected_image_embeddings + time_projected_prompt_embeds

         # ... and by projecting CLIP embeddings into four
         # extra tokens of context that are concatenated to the sequence of outputs from the GLIDE text encoder"
         __UpperCamelCase	:	Optional[int]		    =  self.clip_extra_context_tokens_proj(_lowerCamelCase    )
         __UpperCamelCase	:	Tuple		    =  clip_extra_context_tokens.reshape(_lowerCamelCase     ,			-1     ,			self.clip_extra_context_tokens    )
         __UpperCamelCase	:	Dict		    =  clip_extra_context_tokens.permute(0     ,			2     ,			1    )

         __UpperCamelCase	:	Union[str, Any]		    =  self.encoder_hidden_states_proj(_lowerCamelCase    )
         __UpperCamelCase	:	Optional[Any]		    =  self.text_encoder_hidden_states_norm(_lowerCamelCase    )
         __UpperCamelCase	:	Tuple		    =  torch.cat([clip_extra_context_tokens, text_encoder_hidden_states]     ,			dim=1    )

         return text_encoder_hidden_states, additive_clip_time_embeddings



