

"""simple docstring"""






import os
from glob import glob

import imageio
import torch
import torchvision
import wandb
from img_processing import custom_to_pil, loop_post_process, preprocess, preprocess_vqgan
from loaders import load_vqgan
from PIL import Image
from torch import nn

from transformers import CLIPModel, CLIPTokenizerFast
from utils import get_device, get_timestamp, show_pil







class       snake_case__  :







			def __init__(			self			: Any			,  lowercase			: str = "cpu"			,  lowercase			: str = "openai/clip-vit-large-patch14"  ):





								'''simple docstring'''



								UpperCAmelCase			:   Any      =				device
								UpperCAmelCase			:   Dict      =				CLIPTokenizerFast.from_pretrained(lowercase  )
								UpperCAmelCase			:   Any      =				[0.4_8_1_4_5_4_6_6, 0.4_5_7_8_2_7_5, 0.4_0_8_2_1_0_7_3]
								UpperCAmelCase			:   List[str]      =				[0.2_6_8_6_2_9_5_4, 0.2_6_1_3_0_2_5_8, 0.2_7_5_7_7_7_1_1]
								UpperCAmelCase			:   Dict      =				torchvision.transforms.Normalize(self.image_mean			,  self.image_std  )
								UpperCAmelCase			:   Optional[Any]      =				torchvision.transforms.Resize(2_24  )
								UpperCAmelCase			:   str      =				torchvision.transforms.CenterCrop(2_24  )







			def 			__lowerCAmelCase				(			self			: Union[str, Any]			,  lowercase			: Any  ):





								'''simple docstring'''



								UpperCAmelCase			:   Dict      =				self.resize(lowercase  )
								UpperCAmelCase			:   Dict      =				self.center_crop(lowercase  )
								UpperCAmelCase			:   Optional[Any]      =				self.normalize(lowercase  )
								return images







			def __call__(			self			: int			,  lowercase			: Any=None			,  lowercase			: Optional[int]=None			,  **lowercase			: List[str]  ):





								'''simple docstring'''



								UpperCAmelCase			:   Dict      =				self.tokenizer(text=lowercase			,  **lowercase  )
								UpperCAmelCase			:   Dict      =				self.preprocess_img(lowercase  )
								UpperCAmelCase			:   int      =				{key: value.to(self.device  ) for (key, value) in encoding.items()}
								return encoding







class       snake_case__  ( nn.Module     ):







			def __init__(			self			: Dict			,  lowercase			: Optional[int]=10			,  lowercase			: Union[str, Any]=0.0_1			,  lowercase			: str=None			,  lowercase			: Union[str, Any]=None			,  lowercase			: Any=None			,  lowercase			: int=None			,  lowercase			: List[Any]=None			,  lowercase			: int=None			,  lowercase			: Optional[int]=False			,  lowercase			: int=True			,  lowercase			: str="image"			,  lowercase			: List[str]=True			,  lowercase			: List[Any]=False			,  lowercase			: List[Any]=False			,  lowercase			: Optional[int]=False			,  ):





								'''simple docstring'''



								super().__init__()
								UpperCAmelCase			:   Union[str, Any]      =				None
								UpperCAmelCase			:   Dict      =				device if device else get_device()
								if vqgan:
													UpperCAmelCase			:   Tuple      =				vqgan
								else:
													UpperCAmelCase			:   Optional[Any]      =				load_vqgan(self.device			,  conf_path=lowercase			,  ckpt_path=lowercase  )
								self.vqgan.eval()
								if clip:
													UpperCAmelCase			:   Optional[Any]      =				clip
								else:
													UpperCAmelCase			:   List[Any]      =				CLIPModel.from_pretrained("openai/clip-vit-base-patch32"  )
								self.clip.to(self.device  )
								UpperCAmelCase			:   Union[str, Any]      =				ProcessorGradientFlow(device=self.device  )

								UpperCAmelCase			:   Tuple      =				iterations
								UpperCAmelCase			:   int      =				lr
								UpperCAmelCase			:   Tuple      =				log
								UpperCAmelCase			:   Dict      =				make_grid
								UpperCAmelCase			:   Optional[Any]      =				return_val
								UpperCAmelCase			:   Optional[int]      =				quantize
								UpperCAmelCase			:   List[str]      =				self.vqgan.decoder.z_shape







			def 			__lowerCAmelCase				(			self			: Union[str, Any]			,  lowercase			: Dict=None			,  lowercase			: str=None			,  lowercase			: Union[str, Any]=5			,  lowercase			: Union[str, Any]=True  ):





								'''simple docstring'''



								UpperCAmelCase			:   Union[str, Any]      =				[]
								if output_path is None:
													UpperCAmelCase			:   Optional[Any]      =				"./animation.gif"
								if input_path is None:
													UpperCAmelCase			:   Tuple      =				self.save_path
								UpperCAmelCase			:   List[Any]      =				sorted(glob(input_path + "/*"  )  )
								if not len(lowercase  ):
													raise ValueError(
													    "No images found in save path, aborting (did you pass save_intermediate=True to the generate"
													    " function?)"  )
								if len(lowercase  ) == 1:
													print("Only one image found in save path, (did you pass save_intermediate=True to the generate function?)"  )
								UpperCAmelCase			:   List[str]      =				total_duration / len(lowercase  )
								UpperCAmelCase			:   int      =				[frame_duration] * len(lowercase  )
								if extend_frames:
													UpperCAmelCase			:   Optional[Any]      =				1.5
													UpperCAmelCase			:   List[str]      =				3
								for file_name in paths:
													if file_name.endswith(".png"  ):
																		images.append(imageio.imread(lowercase  )  )
								imageio.mimsave(lowercase			,  lowercase			,  duration=lowercase  )
								print(f"""gif saved to {output_path}"""  )







			def 			__lowerCAmelCase				(			self			: int			,  lowercase			: Optional[int]=None			,  lowercase			: Union[str, Any]=None  ):





								'''simple docstring'''



								if not (path or img):
													raise ValueError("Input either path or tensor"  )
								if img is not None:
													raise NotImplementedError
								UpperCAmelCase			:   str      =				preprocess(Image.open(lowercase  )			,  target_image_size=2_56  ).to(self.device  )
								UpperCAmelCase			:   Any      =				preprocess_vqgan(lowercase  )
								UpperCAmelCase      ,							*UpperCAmelCase			:   Dict      =				self.vqgan.encode(lowercase  )
								return z







			def 			__lowerCAmelCase				(			self			: Optional[int]			,  lowercase			: Dict  ):





								'''simple docstring'''



								UpperCAmelCase			:   str      =				self.latent.detach().requires_grad_()
								UpperCAmelCase			:   int      =				base_latent + transform_vector
								if self.quantize:
													UpperCAmelCase      ,							*UpperCAmelCase			:   Optional[Any]      =				self.vqgan.quantize(lowercase  )
								else:
													UpperCAmelCase			:   Union[str, Any]      =				trans_latent
								return self.vqgan.decode(lowercase  )







			def 			__lowerCAmelCase				(			self			: str			,  lowercase			: str			,  lowercase			: Any			,  lowercase			: List[str]=None  ):





								'''simple docstring'''



								UpperCAmelCase			:   Optional[Any]      =				self.clip_preprocessor(text=lowercase			,  images=lowercase			,  return_tensors="pt"			,  padding=lowercase  )
								UpperCAmelCase			:   Optional[Any]      =				self.clip(**lowercase  )
								UpperCAmelCase			:   Optional[Any]      =				clip_outputs.logits_per_image
								if weights is not None:
													UpperCAmelCase			:   Optional[int]      =				similarity_logits * weights
								return similarity_logits.sum()







			def 			__lowerCAmelCase				(			self			: Optional[int]			,  lowercase			: Dict			,  lowercase			: Union[str, Any]			,  lowercase			: Tuple  ):





								'''simple docstring'''



								UpperCAmelCase			:   Union[str, Any]      =				self._get_clip_similarity(pos_prompts["prompts"]			,  lowercase			,  weights=(1 / pos_prompts["weights"])  )
								if neg_prompts:
													UpperCAmelCase			:   Optional[Any]      =				self._get_clip_similarity(neg_prompts["prompts"]			,  lowercase			,  weights=neg_prompts["weights"]  )
								else:
													UpperCAmelCase			:   Union[str, Any]      =				torch.tensor([1]			,  device=self.device  )
								UpperCAmelCase			:   List[str]      =				-torch.log(lowercase  ) + torch.log(lowercase  )
								return loss







			def 			__lowerCAmelCase				(			self			: Tuple			,  lowercase			: str			,  lowercase			: Dict			,  lowercase			: Any  ):





								'''simple docstring'''



								UpperCAmelCase			:   Union[str, Any]      =				torch.randn_like(self.latent			,  requires_grad=lowercase			,  device=self.device  )
								UpperCAmelCase			:   int      =				torch.optim.Adam([vector]			,  lr=self.lr  )

								for i in range(self.iterations  ):
													optim.zero_grad()
													UpperCAmelCase			:   Tuple      =				self._add_vector(lowercase  )
													UpperCAmelCase			:   List[str]      =				loop_post_process(lowercase  )
													UpperCAmelCase			:   str      =				self._get_CLIP_loss(lowercase			,  lowercase			,  lowercase  )
													print("CLIP loss"			,  lowercase  )
													if self.log:
																		wandb.log({"CLIP Loss": clip_loss}  )
													clip_loss.backward(retain_graph=lowercase  )
													optim.step()
													if self.return_val == "image":
																		yield custom_to_pil(transformed_img[0]  )
													else:
																		yield vector







			def 			__lowerCAmelCase				(			self			: int			,  lowercase			: Tuple			,  lowercase			: List[Any]			,  lowercase			: List[Any]  ):





								'''simple docstring'''



								wandb.init(reinit=lowercase			,  project="face-editor"  )
								wandb.config.update({"Positive Prompts": positive_prompts}  )
								wandb.config.update({"Negative Prompts": negative_prompts}  )
								wandb.config.update({"lr": self.lr, "iterations": self.iterations}  )
								if image_path:
													UpperCAmelCase			:   Tuple      =				Image.open(lowercase  )
													UpperCAmelCase			:   int      =				image.resize((2_56, 2_56)  )
													wandb.log("Original Image"			,  wandb.Image(lowercase  )  )







			def 			__lowerCAmelCase				(			self			: Tuple			,  lowercase			: Optional[Any]  ):





								'''simple docstring'''



								if not prompts:
													return []
								UpperCAmelCase			:   Tuple      =				[]
								UpperCAmelCase			:   str      =				[]
								if isinstance(lowercase			,  lowercase  ):
													UpperCAmelCase			:   Any      =				[prompt.strip() for prompt in prompts.split("|"  )]
								for prompt in prompts:
													if isinstance(lowercase			,  (tuple, list)  ):
																		UpperCAmelCase			:   Optional[Any]      =				prompt[0]
																		UpperCAmelCase			:   List[str]      =				float(prompt[1]  )
													elif ":" in prompt:
																		UpperCAmelCase      ,							UpperCAmelCase			:   List[Any]      =				prompt.split(":"  )
																		UpperCAmelCase			:   str      =				float(lowercase  )
													else:
																		UpperCAmelCase			:   Dict      =				prompt
																		UpperCAmelCase			:   Optional[Any]      =				1.0
													processed_prompts.append(lowercase  )
													weights.append(lowercase  )
								return {
								    "prompts": processed_prompts,
								    "weights": torch.tensor(lowercase			,  device=self.device  ),
								}







			def 			__lowerCAmelCase				(			self			: str			,  lowercase			: List[str]			,  lowercase			: Dict=None			,  lowercase			: str=None			,  lowercase			: Tuple=True			,  lowercase			: List[str]=False			,  lowercase			: List[Any]=True			,  lowercase			: List[str]=True			,  lowercase			: List[str]=None			,  ):





								'''simple docstring'''



								if image_path:
													UpperCAmelCase			:   List[str]      =				self._get_latent(lowercase  )
								else:
													UpperCAmelCase			:   Optional[int]      =				torch.randn(self.latent_dim			,  device=self.device  )
								if self.log:
													self._init_logging(lowercase			,  lowercase			,  lowercase  )

								assert pos_prompts, "You must provide at least one positive prompt."
								UpperCAmelCase			:   Optional[Any]      =				self.process_prompts(lowercase  )
								UpperCAmelCase			:   str      =				self.process_prompts(lowercase  )
								if save_final and save_path is None:
													UpperCAmelCase			:   Union[str, Any]      =				os.path.join("./outputs/"			,  "_".join(pos_prompts["prompts"]  )  )
								if not os.path.exists(lowercase  ):
													os.makedirs(lowercase  )
								else:
													UpperCAmelCase			:   Tuple      =				save_path + "_" + get_timestamp()
													os.makedirs(lowercase  )
								UpperCAmelCase			:   List[Any]      =				save_path

								UpperCAmelCase			:   List[Any]      =				self.vqgan.decode(self.latent  )[0]
								if show_intermediate:
													print("Original Image"  )
													show_pil(custom_to_pil(lowercase  )  )

								UpperCAmelCase			:   List[str]      =				loop_post_process(lowercase  )
								for iter, transformed_img in enumerate(self._optimize_CLIP(lowercase			,  lowercase			,  lowercase  )  ):
													if show_intermediate:
																		show_pil(lowercase  )
													if save_intermediate:
																		transformed_img.save(os.path.join(self.save_path			,  f"""iter_{iter:03d}.png"""  )  )
													if self.log:
																		wandb.log({"Image": wandb.Image(lowercase  )}  )
								if show_final:
													show_pil(lowercase  )
								if save_final:
													transformed_img.save(os.path.join(self.save_path			,  f"""iter_{iter:03d}_final.png"""  )  )





"""simple docstring"""






import inspect
import unittest

from huggingface_hub import hf_hub_download

from transformers import ASTConfig
from transformers.testing_utils import require_torch, require_torchaudio, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_torchaudio_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
	import torch
	from torch import nn

	from transformers import ASTForAudioClassification, ASTModel
	from transformers.models.audio_spectrogram_transformer.modeling_audio_spectrogram_transformer import (
	    AUDIO_SPECTROGRAM_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST,
	)


if is_torchaudio_available():
	import torchaudio

	from transformers import ASTFeatureExtractor







class       snake_case__  :







			def __init__(			self			: Optional[Any]			,  lowercase			: int			,  lowercase			: List[Any]=13			,  lowercase			: List[str]=2			,  lowercase			: Tuple=24			,  lowercase			: Dict=16			,  lowercase			: Optional[int]=True			,  lowercase			: Any=True			,  lowercase			: Dict=32			,  lowercase			: List[str]=5			,  lowercase			: Union[str, Any]=4			,  lowercase			: Tuple=37			,  lowercase			: Tuple="gelu"			,  lowercase			: Dict=0.1			,  lowercase			: Any=0.1			,  lowercase			: Any=10			,  lowercase			: List[str]=0.0_2			,  lowercase			: int=None			,  lowercase			: Optional[Any]=2			,  lowercase			: str=2			,  ):





								'''simple docstring'''



								UpperCAmelCase			:   int      =				parent
								UpperCAmelCase			:   Optional[int]      =				batch_size
								UpperCAmelCase			:   str      =				patch_size
								UpperCAmelCase			:   Dict      =				max_length
								UpperCAmelCase			:   Dict      =				num_mel_bins
								UpperCAmelCase			:   Optional[Any]      =				is_training
								UpperCAmelCase			:   str      =				use_labels
								UpperCAmelCase			:   Dict      =				hidden_size
								UpperCAmelCase			:   Tuple      =				num_hidden_layers
								UpperCAmelCase			:   Tuple      =				num_attention_heads
								UpperCAmelCase			:   Optional[Any]      =				intermediate_size
								UpperCAmelCase			:   Tuple      =				hidden_act
								UpperCAmelCase			:   Optional[int]      =				hidden_dropout_prob
								UpperCAmelCase			:   Optional[Any]      =				attention_probs_dropout_prob
								UpperCAmelCase			:   Any      =				type_sequence_label_size
								UpperCAmelCase			:   Dict      =				initializer_range
								UpperCAmelCase			:   Optional[int]      =				scope
								UpperCAmelCase			:   List[Any]      =				frequency_stride
								UpperCAmelCase			:   Tuple      =				time_stride

								# in AST, the seq length equals the number of patches + 2 (we add 2 for the [CLS] and distillation tokens)
								UpperCAmelCase			:   List[str]      =				(self.num_mel_bins - self.patch_size) // self.frequency_stride + 1
								UpperCAmelCase			:   List[str]      =				(self.max_length - self.patch_size) // self.time_stride + 1
								UpperCAmelCase			:   Dict      =				frequency_out_dimension * time_out_dimension
								UpperCAmelCase			:   int      =				num_patches + 2







			def 			__lowerCAmelCase				(			self			: Optional[Any]  ):





								'''simple docstring'''



								UpperCAmelCase			:   List[str]      =				floats_tensor([self.batch_size, self.max_length, self.num_mel_bins]  )

								UpperCAmelCase			:   Optional[Any]      =				None
								if self.use_labels:
													UpperCAmelCase			:   List[str]      =				ids_tensor([self.batch_size]			,  self.type_sequence_label_size  )

								UpperCAmelCase			:   List[str]      =				self.get_config()

								return config, input_values, labels







			def 			__lowerCAmelCase				(			self			: str  ):





								'''simple docstring'''



								return ASTConfig(
								    patch_size=self.patch_size			,  max_length=self.max_length			,  num_mel_bins=self.num_mel_bins			,  hidden_size=self.hidden_size			,  num_hidden_layers=self.num_hidden_layers			,  num_attention_heads=self.num_attention_heads			,  intermediate_size=self.intermediate_size			,  hidden_act=self.hidden_act			,  hidden_dropout_prob=self.hidden_dropout_prob			,  attention_probs_dropout_prob=self.attention_probs_dropout_prob			,  is_decoder=lowercase			,  initializer_range=self.initializer_range			,  frequency_stride=self.frequency_stride			,  time_stride=self.time_stride			,  )







			def 			__lowerCAmelCase				(			self			: List[Any]			,  lowercase			: Optional[int]			,  lowercase			: Any			,  lowercase			: List[str]  ):





								'''simple docstring'''



								UpperCAmelCase			:   Tuple      =				ASTModel(config=lowercase  )
								model.to(lowercase  )
								model.eval()
								UpperCAmelCase			:   Optional[int]      =				model(lowercase  )
								self.parent.assertEqual(result.last_hidden_state.shape			,  (self.batch_size, self.seq_length, self.hidden_size)  )







			def 			__lowerCAmelCase				(			self			: Dict  ):





								'''simple docstring'''



								UpperCAmelCase			:   Union[str, Any]      =				self.prepare_config_and_inputs()
								(
								    (
								    UpperCAmelCase
								)      ,							(
								    UpperCAmelCase
								)      ,							(
								    UpperCAmelCase
								)      ,							
								)			:   Tuple      =				config_and_inputs
								UpperCAmelCase			:   int      =				{"input_values": input_values}
								return config, inputs_dict







@require_torch
class       snake_case__  ( lowerCAmelCase_					,  lowerCAmelCase_					,  unittest.TestCase     ):
			SCREAMING_SNAKE_CASE__     				=						(
			    (
			        ASTModel,
			        ASTForAudioClassification,
			    )
			    if is_torch_available()
			    else ()
			)
			SCREAMING_SNAKE_CASE__     				=						(
			    {'''audio-classification''': ASTForAudioClassification, '''feature-extraction''': ASTModel}
			    if is_torch_available()
			    else {}
			)
			SCREAMING_SNAKE_CASE__     				=						False
			SCREAMING_SNAKE_CASE__     				=						False
			SCREAMING_SNAKE_CASE__     				=						False
			SCREAMING_SNAKE_CASE__     				=						False







			def 			__lowerCAmelCase				(			self			: Optional[Any]			,  lowercase			: Optional[Any]			,  lowercase			: Union[str, Any]			,  lowercase			: List[str]			,  lowercase			: Any			,  lowercase			: str  ):





								'''simple docstring'''



								if pipeline_test_casse_name == "AudioClassificationPipelineTests":
													return True

								return False







			def 			__lowerCAmelCase				(			self			: str  ):





								'''simple docstring'''



								UpperCAmelCase			:   int      =				ASTModelTester(self  )
								UpperCAmelCase			:   Union[str, Any]      =				ConfigTester(self			,  config_class=lowercase			,  has_text_modality=lowercase			,  hidden_size=37  )







			def 			__lowerCAmelCase				(			self			: str  ):





								'''simple docstring'''



								self.config_tester.run_common_tests()







			@unittest.skip(reason="AST does not use inputs_embeds"  )
			def 			__lowerCAmelCase				(			self			: List[str]  ):





								'''simple docstring'''



								pass







			def 			__lowerCAmelCase				(			self			: Dict  ):





								'''simple docstring'''



								UpperCAmelCase      ,							UpperCAmelCase			:   str      =				self.model_tester.prepare_config_and_inputs_for_common()

								for model_class in self.all_model_classes:
													UpperCAmelCase			:   Dict      =				model_class(lowercase  )
													self.assertIsInstance(model.get_input_embeddings()			,  (nn.Module)  )
													UpperCAmelCase			:   List[Any]      =				model.get_output_embeddings()
													self.assertTrue(x is None or isinstance(lowercase			,  nn.Linear  )  )







			def 			__lowerCAmelCase				(			self			: List[Any]  ):





								'''simple docstring'''



								UpperCAmelCase      ,							UpperCAmelCase			:   List[str]      =				self.model_tester.prepare_config_and_inputs_for_common()

								for model_class in self.all_model_classes:
													UpperCAmelCase			:   Dict      =				model_class(lowercase  )
													UpperCAmelCase			:   Any      =				inspect.signature(model.forward  )
													# signature.parameters is an OrderedDict => so arg_names order is deterministic
													UpperCAmelCase			:   List[str]      =				[*signature.parameters.keys()]

													UpperCAmelCase			:   List[str]      =				["input_values"]
													self.assertListEqual(arg_names[:1]			,  lowercase  )







			def 			__lowerCAmelCase				(			self			: Optional[int]  ):





								'''simple docstring'''



								UpperCAmelCase			:   List[str]      =				self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_model(*lowercase  )







			@slow
			def 			__lowerCAmelCase				(			self			: str  ):





								'''simple docstring'''



								for model_name in AUDIO_SPECTROGRAM_TRANSFORMER_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
													UpperCAmelCase			:   Dict      =				ASTModel.from_pretrained(lowercase  )
													self.assertIsNotNone(lowercase  )



def   lowercase_	(    ):




					'''simple docstring'''






					UpperCAmelCase			:   List[str]      =				hf_hub_download(
					    repo_id="nielsr/audio-spectogram-transformer-checkpoint"							,      filename="sample_audio.flac"							,      repo_type="dataset"	)

					UpperCAmelCase      ,							UpperCAmelCase			:   str      =				torchaudio.load(_lowercase	)

					return audio, sampling_rate







@require_torch
@require_torchaudio
class       snake_case__  ( unittest.TestCase     ):







			@cached_property
			def 			__lowerCAmelCase				(			self			: List[Any]  ):





								'''simple docstring'''



								return (
								    ASTFeatureExtractor.from_pretrained("MIT/ast-finetuned-audioset-10-10-0.4593"  )
								    if is_torchaudio_available()
								    else None
								)







			@slow
			def 			__lowerCAmelCase				(			self			: Union[str, Any]  ):





								'''simple docstring'''



								UpperCAmelCase			:   Union[str, Any]      =				self.default_feature_extractor
								UpperCAmelCase			:   Union[str, Any]      =				ASTForAudioClassification.from_pretrained("MIT/ast-finetuned-audioset-10-10-0.4593"  ).to(lowercase  )

								UpperCAmelCase			:   Any      =				self.default_feature_extractor
								UpperCAmelCase      ,							UpperCAmelCase			:   Optional[int]      =				prepare_audio()
								UpperCAmelCase			:   str      =				audio.squeeze().numpy()
								UpperCAmelCase			:   List[Any]      =				feature_extractor(lowercase			,  sampling_rate=lowercase			,  return_tensors="pt"  ).to(lowercase  )

								# forward pass
								with torch.no_grad():
													UpperCAmelCase			:   Dict      =				model(**lowercase  )

								# verify the logits
								UpperCAmelCase			:   Dict      =				torch.Size((1, 5_27)  )
								self.assertEqual(outputs.logits.shape			,  lowercase  )

								UpperCAmelCase			:   Optional[Any]      =				torch.tensor([-0.8_7_6_0, -7.0_0_4_2, -8.6_6_0_2]  ).to(lowercase  )

								self.assertTrue(torch.allclose(outputs.logits[0, :3]			,  lowercase			,  atol=1E-4  )  )


