





import os
import re
import warnings
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
					from .tokenization_ta import TaTokenizer
else:
					UpperCAmelCase_ 							=       None


UpperCAmelCase_ 							=       logging.get_logger(__name__)

UpperCAmelCase_ 							=       {"""vocab_file""": """spiece.model""", """tokenizer_file""": """tokenizer.json"""}

UpperCAmelCase_ 							=       {
    """vocab_file""": {
        """t5-small""": """https://huggingface.co/t5-small/resolve/main/spiece.model""",
        """t5-base""": """https://huggingface.co/t5-base/resolve/main/spiece.model""",
        """t5-large""": """https://huggingface.co/t5-large/resolve/main/spiece.model""",
        """t5-3b""": """https://huggingface.co/t5-3b/resolve/main/spiece.model""",
        """t5-11b""": """https://huggingface.co/t5-11b/resolve/main/spiece.model""",
    },
    """tokenizer_file""": {
        """t5-small""": """https://huggingface.co/t5-small/resolve/main/tokenizer.json""",
        """t5-base""": """https://huggingface.co/t5-base/resolve/main/tokenizer.json""",
        """t5-large""": """https://huggingface.co/t5-large/resolve/main/tokenizer.json""",
        """t5-3b""": """https://huggingface.co/t5-3b/resolve/main/tokenizer.json""",
        """t5-11b""": """https://huggingface.co/t5-11b/resolve/main/tokenizer.json""",
    },
}


# TODO(PVP) - this should be removed in Transformers v5
UpperCAmelCase_ 							=       {
    """t5-small""": 512,
    """t5-base""": 512,
    """t5-large""": 512,
    """t5-3b""": 512,
    """t5-11b""": 512,
}
class      UpperCamelCase_						(					_lowerCamelCase   ):
					lowerCAmelCase_							=		VOCAB_FILES_NAMES
					lowerCAmelCase_							=		PRETRAINED_VOCAB_FILES_MAP
					lowerCAmelCase_							=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
					lowerCAmelCase_							=		['''input_ids''', '''attention_mask''']
					lowerCAmelCase_							=		TaTokenizer

					lowerCAmelCase_							=		[]




					def __init__(						self			,  lowerCAmelCase_=None			,  lowerCAmelCase_=None			,  lowerCAmelCase_="</s>"			,  lowerCAmelCase_="<unk>"			,  lowerCAmelCase_="<pad>"			,  lowerCAmelCase_=100			,  lowerCAmelCase_=None			,  **lowerCAmelCase_			,  )						->							Dict:
										# Add extra_ids to the special token list
										if extra_ids > 0 and additional_special_tokens is None:
															_snake_case													=			[F'''<extra_id_{i}>''' for i in range(lowerCAmelCase_					)]
										elif extra_ids > 0 and additional_special_tokens is not None:
															# Check that we have the right number of extra special tokens
															_snake_case													=			len(set(filter(lambda lowerCAmelCase_					: bool('extra_id_' in str(lowerCAmelCase_					)					)			,  lowerCAmelCase_					)					)					)
															if extra_tokens != extra_ids:
																				raise ValueError(
																				    F'''Both extra_ids ({extra_ids}) and additional_special_tokens ({additional_special_tokens}) are'''
																				    ' provided to T5Tokenizer. In this case the additional_special_tokens must include the extra_ids'
																				    ' tokens'					)

										super().__init__(
										    lowerCAmelCase_			,  tokenizer_file=lowerCAmelCase_			,  eos_token=lowerCAmelCase_			,  unk_token=lowerCAmelCase_			,  pad_token=lowerCAmelCase_			,  extra_ids=lowerCAmelCase_			,  additional_special_tokens=lowerCAmelCase_			,  **lowerCAmelCase_			,  )

										_snake_case													=			vocab_file
										_snake_case													=			False if not self.vocab_file else True
										_snake_case													=			extra_ids




					@staticmethod
					def 		lowerCAmelCase     (						lowerCAmelCase_			,  lowerCAmelCase_			,  lowerCAmelCase_					)						->							List[str]:
										if pretrained_model_name_or_path in TaTokenizerFast.max_model_input_sizes:
															_snake_case													=			TaTokenizerFast.max_model_input_sizes[pretrained_model_name_or_path]
															if init_max_model_length is not None and init_max_model_length != max_model_length:
																				return init_max_model_length
															elif init_max_model_length is None:
																				warnings.warn(
																				    'This tokenizer was incorrectly instantiated with a model max length of'
																				    F''' {deprecated_max_model_length} which will be corrected in Transformers v5.\nFor now, this'''
																				    ' behavior is kept to avoid breaking backwards compatibility when padding/encoding with'
																				    ' `truncation is True`.\n- Be aware that you SHOULD NOT rely on'
																				    F''' {pretrained_model_name_or_path} automatically truncating your input to'''
																				    F''' {deprecated_max_model_length} when padding/encoding.\n- If you want to encode/pad to sequences'''
																				    F''' longer than {deprecated_max_model_length} you can either instantiate this tokenizer with'''
																				    ' `model_max_length` or pass `max_length` when encoding/padding.\n- To avoid this warning, please'
																				    ' instantiate this tokenizer with `model_max_length` set to your preferred value.'			,  lowerCAmelCase_			,  )

										return max_model_length




					def 		lowerCAmelCase     (						self			,  lowerCAmelCase_			,  lowerCAmelCase_ = None					)						->							Tuple[str]:
										if not self.can_save_slow_tokenizer:
															raise ValueError(
															    'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
															    'tokenizer.'					)

										if not os.path.isdir(lowerCAmelCase_					):
															logger.error(F'''Vocabulary path ({save_directory}) should be a directory'''					)
															return
										_snake_case													=			os.path.join(
										    lowerCAmelCase_			,  (filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']					)

										if os.path.abspath(self.vocab_file					) != os.path.abspath(lowerCAmelCase_					):
															copyfile(self.vocab_file			,  lowerCAmelCase_					)
															logger.info(F'''Copy vocab file to {out_vocab_file}'''					)

										return (out_vocab_file,)




					def 		lowerCAmelCase     (						self			,  lowerCAmelCase_			,  lowerCAmelCase_ = None					)						->							List[int]:
										_snake_case													=			token_ids_a + [self.eos_token_id]
										if token_ids_a is None:
															return self.prefix_tokens + token_ids_a
										else:
															_snake_case													=			token_ids_a + [self.eos_token_id]
															return self.prefix_tokens + token_ids_a + token_ids_a




					def 		lowerCAmelCase     (						self			,  lowerCAmelCase_			,  lowerCAmelCase_ = None					)						->							List[int]:
										_snake_case													=			[self.eos_token_id]

										if token_ids_a is None:
															return len(token_ids_a + eos					) * [0]
										return len(token_ids_a + eos + token_ids_a + eos					) * [0]




					def 		lowerCAmelCase     (						self					)						->							Any:
										return list(
										    set(filter(lambda lowerCAmelCase_					: bool(re.search(r'<extra_id_\d+>'			,  lowerCAmelCase_					)					) is not None			,  self.additional_special_tokens					)					)					)






					def 		lowerCAmelCase     (						self					)						->							Dict:
										return [self.convert_tokens_to_ids(lowerCAmelCase_					) for token in self.get_sentinel_tokens()]










import math

import torch
from torch import nn

from ..configuration_utils import ConfigMixin, register_to_config
from .attention_processor import Attention
from .embeddings import get_timestep_embedding
from .modeling_utils import ModelMixin
class      UpperCamelCase_						(					_lowerCamelCase							,					_lowerCamelCase   ):




					@register_to_config
					def __init__(						self			,  lowerCAmelCase_ = 128			,  lowerCAmelCase_ = 256			,  lowerCAmelCase_ = 20_00.0			,  lowerCAmelCase_ = 768			,  lowerCAmelCase_ = 12			,  lowerCAmelCase_ = 12			,  lowerCAmelCase_ = 64			,  lowerCAmelCase_ = 2048			,  lowerCAmelCase_ = 0.1			,  )						->							Union[str, Any]:
										super().__init__()

										_snake_case													=			nn.Sequential(
										    nn.Linear(lowerCAmelCase_			,  d_model * 4			,  bias=lowerCAmelCase_					)			,  nn.SiLU()			,  nn.Linear(d_model * 4			,  d_model * 4			,  bias=lowerCAmelCase_					)			,  nn.SiLU()			,  )

										_snake_case													=			nn.Embedding(lowerCAmelCase_			,  lowerCAmelCase_					)
										_snake_case													=			False

										_snake_case													=			nn.Linear(lowerCAmelCase_			,  lowerCAmelCase_			,  bias=lowerCAmelCase_					)

										_snake_case													=			nn.Dropout(p=lowerCAmelCase_					)

										_snake_case													=			nn.ModuleList()
										for lyr_num in range(lowerCAmelCase_					):
															# FiLM conditional T5 decoder
															_snake_case													=			DecoderLayer(d_model=lowerCAmelCase_			,  d_kv=lowerCAmelCase_			,  num_heads=lowerCAmelCase_			,  d_ff=lowerCAmelCase_			,  dropout_rate=lowerCAmelCase_					)
															self.decoders.append(lowerCAmelCase_					)

										_snake_case													=			TaLayerNorm(lowerCAmelCase_					)

										_snake_case													=			nn.Dropout(p=lowerCAmelCase_					)
										_snake_case													=			nn.Linear(lowerCAmelCase_			,  lowerCAmelCase_			,  bias=lowerCAmelCase_					)




					def 		lowerCAmelCase     (						self			,  lowerCAmelCase_			,  lowerCAmelCase_					)						->							Tuple:
										_snake_case													=			torch.mul(query_input.unsqueeze(-1					)			,  key_input.unsqueeze(-2					)					)
										return mask.unsqueeze(-3					)






					def 		lowerCAmelCase     (						self			,  lowerCAmelCase_			,  lowerCAmelCase_			,  lowerCAmelCase_					)						->							Optional[Any]:
										_snake_case							,       _snake_case							,       _snake_case													=			decoder_input_tokens.shape
										assert decoder_noise_time.shape == (batch,)

										# decoder_noise_time is in [0, 1), so rescale to expected timing range.
										_snake_case													=			get_timestep_embedding(
										    decoder_noise_time * self.config.max_decoder_noise_time			,  embedding_dim=self.config.d_model			,  max_period=self.config.max_decoder_noise_time			,  ).to(dtype=self.dtype					)

										_snake_case													=			self.conditioning_emb(lowerCAmelCase_					).unsqueeze(1					)

										assert conditioning_emb.shape == (batch, 1, self.config.d_model * 4)

										_snake_case													=			decoder_input_tokens.shape[1]

										# If we want to use relative positions for audio context, we can just offset
										# this sequence by the length of encodings_and_masks.
										_snake_case													=			torch.broadcast_to(
										    torch.arange(lowerCAmelCase_			,  device=decoder_input_tokens.device					)			,  (batch, seq_length)			,  )

										_snake_case													=			self.position_encoding(lowerCAmelCase_					)

										_snake_case													=			self.continuous_inputs_projection(lowerCAmelCase_					)
										inputs += position_encodings
										_snake_case													=			self.dropout(lowerCAmelCase_					)

										# decoder: No padding present.
										_snake_case													=			torch.ones(
										    decoder_input_tokens.shape[:2]			,  device=decoder_input_tokens.device			,  dtype=inputs.dtype					)

										# Translate encoding masks to encoder-decoder masks.
										_snake_case													=			[(x, self.encoder_decoder_mask(lowerCAmelCase_			,  lowerCAmelCase_					)) for x, y in encodings_and_masks]

										# cross attend style: concat encodings
										_snake_case													=			torch.cat([x[0] for x in encodings_and_encdec_masks]			,  dim=1					)
										_snake_case													=			torch.cat([x[1] for x in encodings_and_encdec_masks]			,  dim=-1					)

										for lyr in self.decoders:
															_snake_case													=			lyr(
															    lowerCAmelCase_			,  conditioning_emb=lowerCAmelCase_			,  encoder_hidden_states=lowerCAmelCase_			,  encoder_attention_mask=lowerCAmelCase_			,  )[0]

										_snake_case													=			self.decoder_norm(lowerCAmelCase_					)
										_snake_case													=			self.post_dropout(lowerCAmelCase_					)

										_snake_case													=			self.spec_out(lowerCAmelCase_					)
										return spec_out





class      UpperCamelCase_						(					nn.Module   ):




					def __init__(						self			,  lowerCAmelCase_			,  lowerCAmelCase_			,  lowerCAmelCase_			,  lowerCAmelCase_			,  lowerCAmelCase_			,  lowerCAmelCase_=1E-6					)						->							Tuple:
										super().__init__()
										_snake_case													=			nn.ModuleList()

										# cond self attention: layer 0
										self.layer.append(
										    TaLayerSelfAttentionCond(d_model=lowerCAmelCase_			,  d_kv=lowerCAmelCase_			,  num_heads=lowerCAmelCase_			,  dropout_rate=lowerCAmelCase_					)					)

										# cross attention: layer 1
										self.layer.append(
										    TaLayerCrossAttention(
										        d_model=lowerCAmelCase_			,  d_kv=lowerCAmelCase_			,  num_heads=lowerCAmelCase_			,  dropout_rate=lowerCAmelCase_			,  layer_norm_epsilon=lowerCAmelCase_			,  )					)

										# Film Cond MLP + dropout: last layer
										self.layer.append(
										    TaLayerFFCond(d_model=lowerCAmelCase_			,  d_ff=lowerCAmelCase_			,  dropout_rate=lowerCAmelCase_			,  layer_norm_epsilon=lowerCAmelCase_					)					)






					def 		lowerCAmelCase     (						self			,  lowerCAmelCase_			,  lowerCAmelCase_=None			,  lowerCAmelCase_=None			,  lowerCAmelCase_=None			,  lowerCAmelCase_=None			,  lowerCAmelCase_=None			,  )						->							Tuple:
										_snake_case													=			self.layer[0](
										    lowerCAmelCase_			,  conditioning_emb=lowerCAmelCase_			,  attention_mask=lowerCAmelCase_			,  )

										if encoder_hidden_states is not None:
															_snake_case													=			torch.where(encoder_attention_mask > 0			,  0			,  -1E10					).to(
															    encoder_hidden_states.dtype					)

															_snake_case													=			self.layer[1](
															    lowerCAmelCase_			,  key_value_states=lowerCAmelCase_			,  attention_mask=lowerCAmelCase_			,  )

										# Apply Film Conditional Feed Forward layer
										_snake_case													=			self.layer[-1](lowerCAmelCase_			,  lowerCAmelCase_					)

										return (hidden_states,)





class      UpperCamelCase_						(					nn.Module   ):




					def __init__(						self			,  lowerCAmelCase_			,  lowerCAmelCase_			,  lowerCAmelCase_			,  lowerCAmelCase_					)						->							Tuple:
										super().__init__()
										_snake_case													=			TaLayerNorm(lowerCAmelCase_					)
										_snake_case													=			TaFiLMLayer(in_features=d_model * 4			,  out_features=lowerCAmelCase_					)
										_snake_case													=			Attention(query_dim=lowerCAmelCase_			,  heads=lowerCAmelCase_			,  dim_head=lowerCAmelCase_			,  out_bias=lowerCAmelCase_			,  scale_qk=lowerCAmelCase_					)
										_snake_case													=			nn.Dropout(lowerCAmelCase_					)






					def 		lowerCAmelCase     (						self			,  lowerCAmelCase_			,  lowerCAmelCase_=None			,  lowerCAmelCase_=None			,  )						->							str:
										# pre_self_attention_layer_norm
										_snake_case													=			self.layer_norm(lowerCAmelCase_					)

										if conditioning_emb is not None:
															_snake_case													=			self.FiLMLayer(lowerCAmelCase_			,  lowerCAmelCase_					)

										# Self-attention block
										_snake_case													=			self.attention(lowerCAmelCase_					)

										_snake_case													=			hidden_states + self.dropout(lowerCAmelCase_					)

										return hidden_states





class      UpperCamelCase_						(					nn.Module   ):




					def __init__(						self			,  lowerCAmelCase_			,  lowerCAmelCase_			,  lowerCAmelCase_			,  lowerCAmelCase_			,  lowerCAmelCase_					)						->							str:
										super().__init__()
										_snake_case													=			Attention(query_dim=lowerCAmelCase_			,  heads=lowerCAmelCase_			,  dim_head=lowerCAmelCase_			,  out_bias=lowerCAmelCase_			,  scale_qk=lowerCAmelCase_					)
										_snake_case													=			TaLayerNorm(lowerCAmelCase_			,  eps=lowerCAmelCase_					)
										_snake_case													=			nn.Dropout(lowerCAmelCase_					)






					def 		lowerCAmelCase     (						self			,  lowerCAmelCase_			,  lowerCAmelCase_=None			,  lowerCAmelCase_=None			,  )						->							Dict:
										_snake_case													=			self.layer_norm(lowerCAmelCase_					)
										_snake_case													=			self.attention(
										    lowerCAmelCase_			,  encoder_hidden_states=lowerCAmelCase_			,  attention_mask=attention_mask.squeeze(1					)			,  )
										_snake_case													=			hidden_states + self.dropout(lowerCAmelCase_					)
										return layer_output





class      UpperCamelCase_						(					nn.Module   ):




					def __init__(						self			,  lowerCAmelCase_			,  lowerCAmelCase_			,  lowerCAmelCase_			,  lowerCAmelCase_					)						->							List[str]:
										super().__init__()
										_snake_case													=			TaDenseGatedActDense(d_model=lowerCAmelCase_			,  d_ff=lowerCAmelCase_			,  dropout_rate=lowerCAmelCase_					)
										_snake_case													=			TaFiLMLayer(in_features=d_model * 4			,  out_features=lowerCAmelCase_					)
										_snake_case													=			TaLayerNorm(lowerCAmelCase_			,  eps=lowerCAmelCase_					)
										_snake_case													=			nn.Dropout(lowerCAmelCase_					)






					def 		lowerCAmelCase     (						self			,  lowerCAmelCase_			,  lowerCAmelCase_=None					)						->							Union[str, Any]:
										_snake_case													=			self.layer_norm(lowerCAmelCase_					)
										if conditioning_emb is not None:
															_snake_case													=			self.film(lowerCAmelCase_			,  lowerCAmelCase_					)

										_snake_case													=			self.DenseReluDense(lowerCAmelCase_					)
										_snake_case													=			hidden_states + self.dropout(lowerCAmelCase_					)
										return hidden_states





class      UpperCamelCase_						(					nn.Module   ):




					def __init__(						self			,  lowerCAmelCase_			,  lowerCAmelCase_			,  lowerCAmelCase_					)						->							Union[str, Any]:
										super().__init__()
										_snake_case													=			nn.Linear(lowerCAmelCase_			,  lowerCAmelCase_			,  bias=lowerCAmelCase_					)
										_snake_case													=			nn.Linear(lowerCAmelCase_			,  lowerCAmelCase_			,  bias=lowerCAmelCase_					)
										_snake_case													=			nn.Linear(lowerCAmelCase_			,  lowerCAmelCase_			,  bias=lowerCAmelCase_					)
										_snake_case													=			nn.Dropout(lowerCAmelCase_					)
										_snake_case													=			NewGELUActivation()






					def 		lowerCAmelCase     (						self			,  lowerCAmelCase_					)						->							Any:
										_snake_case													=			self.act(self.wi_a(lowerCAmelCase_					)					)
										_snake_case													=			self.wi_a(lowerCAmelCase_					)
										_snake_case													=			hidden_gelu * hidden_linear
										_snake_case													=			self.dropout(lowerCAmelCase_					)

										_snake_case													=			self.wo(lowerCAmelCase_					)
										return hidden_states





class      UpperCamelCase_						(					nn.Module   ):




					def __init__(						self			,  lowerCAmelCase_			,  lowerCAmelCase_=1E-6					)						->							str:
										super().__init__()
										_snake_case													=			nn.Parameter(torch.ones(lowerCAmelCase_					)					)
										_snake_case													=			eps






					def 		lowerCAmelCase     (						self			,  lowerCAmelCase_					)						->							int:
										# T5 uses a layer_norm which only scales and doesn't shift, which is also known as Root Mean
										# Square Layer Normalization https://arxiv.org/abs/1910.07467 thus variance is calculated
										# w/o mean and there is no bias. Additionally we want to make sure that the accumulation for
										# half-precision inputs is done in fp32

										_snake_case													=			hidden_states.to(torch.floataa					).pow(2					).mean(-1			,  keepdim=lowerCAmelCase_					)
										_snake_case													=			hidden_states * torch.rsqrt(variance + self.variance_epsilon					)

										# convert into half-precision if necessary
										if self.weight.dtype in [torch.floataa, torch.bfloataa]:
															_snake_case													=			hidden_states.to(self.weight.dtype					)

										return self.weight * hidden_states





class      UpperCamelCase_						(					nn.Module   ):




					def 		lowerCAmelCase     (						self			,  lowerCAmelCase_					)						->							torch.Tensor:
										return 0.5 * input * (1.0 + torch.tanh(math.sqrt(2.0 / math.pi					) * (input + 0.04_47_15 * torch.pow(lowerCAmelCase_			,  3.0					))					))





class      UpperCamelCase_						(					nn.Module   ):




					def __init__(						self			,  lowerCAmelCase_			,  lowerCAmelCase_					)						->							Any:
										super().__init__()
										_snake_case													=			nn.Linear(lowerCAmelCase_			,  out_features * 2			,  bias=lowerCAmelCase_					)






					def 		lowerCAmelCase     (						self			,  lowerCAmelCase_			,  lowerCAmelCase_					)						->							Optional[Any]:
										_snake_case													=			self.scale_bias(lowerCAmelCase_					)
										_snake_case							,       _snake_case													=			torch.chunk(lowerCAmelCase_			,  2			,  -1					)
										_snake_case													=			x * (1 + scale) + shift
										return x



