



import baseaa
import io
import json
import os
from copy import deepcopy

from ..optimizer import AcceleratedOptimizer
from ..scheduler import AcceleratedScheduler







class        lowerCAmelCase__      :

						def __init__(			self       ,							a				)					->      Tuple:


								'''simple docstring'''

								if isinstance(a       ,							a				):
										# Don't modify user's data should they want to reuse it (e.g. in tests), because once we
										# modified it, it will not be accepted here again, since `auto` values would have been overridden
										_UpperCamelCase						    =	deepcopy(a				)
								elif os.path.exists(a				):
										with io.open(a       ,							"""r"""       ,							encoding="""utf-8"""				) as f:
												_UpperCamelCase						    =	json.load(a				)
								else:
										try:
												_UpperCamelCase						    =	baseaa.urlsafe_baadecode(a				).decode("""utf-8"""				)
												_UpperCamelCase						    =	json.loads(a				)
										except (UnicodeDecodeError, AttributeError, ValueError):
												raise ValueError(
												    F'Expected a string path to an existing deepspeed config, or a dictionary, or a base64 encoded string. Received: {config_file_or_dict}'				)

								_UpperCamelCase						    =	config

								self.set_stage_and_offload()

						def  A_				(			self				)					->      Dict:


								'''simple docstring'''

								_UpperCamelCase						    =	self.get_value("""zero_optimization.stage"""       ,							-1				)

								# offload
								_UpperCamelCase						    =	False
								if self.is_zeroa() or self.is_zeroa():
										_UpperCamelCase						    =	set(["""cpu""", """nvme"""]				)
										_UpperCamelCase						    =	set(
										    [
										        self.get_value("""zero_optimization.offload_optimizer.device"""				),
										        self.get_value("""zero_optimization.offload_param.device"""				),
										    ]				)
										if len(offload_devices & offload_devices_valid				) > 0:
												_UpperCamelCase						    =	True

						def  A_				(			self       ,							a				)					->      Dict:


								'''simple docstring'''

								_UpperCamelCase						    =	self.config

								# find the config node of interest if it exists
								_UpperCamelCase						    =	ds_key_long.split("""."""				)
								_UpperCamelCase						    =	nodes.pop()
								for node in nodes:
										_UpperCamelCase						    =	config.get(a				)
										if config is None:
												return None, ds_key

								return config, ds_key

						def  A_				(			self       ,							a       ,							a=None				)					->      Dict:


								'''simple docstring'''

								_UpperCamelCase    ,	_UpperCamelCase						    =	self.find_config_node(a				)
								if config is None:
										return default
								return config.get(a       ,							a				)

						def  A_				(			self       ,							a       ,							a=False				)					->      List[str]:


								'''simple docstring'''

								_UpperCamelCase						    =	self.config

								# find the config node of interest if it exists
								_UpperCamelCase						    =	ds_key_long.split("""."""				)
								for node in nodes:
										_UpperCamelCase						    =	config
										_UpperCamelCase						    =	config.get(a				)
										if config is None:
												if must_exist:
														raise ValueError(F'Can\'t find {ds_key_long} entry in the config: {self.config}'				)
												else:
														return

        # if found remove it
								if parent_config is not None:
										parent_config.pop(a				)

						def  A_				(			self       ,							a				)					->      int:


								'''simple docstring'''

								_UpperCamelCase						    =	self.get_value(a				)
								return False if value is None else bool(a				)

						def  A_				(			self       ,							a				)					->      Optional[int]:


								'''simple docstring'''

								_UpperCamelCase						    =	self.get_value(a				)
								return False if value is None else not bool(a				)

						def  A_				(			self				)					->      Tuple:


								'''simple docstring'''

								return self._stage == 2

						def  A_				(			self				)					->      List[Any]:


								'''simple docstring'''

								return self._stage == 3







						def  A_				(			self				)					->      Dict:


								'''simple docstring'''

								return self._offload







class        lowerCAmelCase__      :

						def __init__(			self       ,							a				)					->      Optional[Any]:


								'''simple docstring'''

								_UpperCamelCase						    =	engine







						def  A_				(			self       ,							a       ,							**a				)					->      str:


								'''simple docstring'''

								self.engine.backward(a       ,							**a				)

								# Deepspeed's `engine.step` performs the following operations:
								# - gradient accumulation check
								# - gradient clipping
								# - optimizer step
								# - zero grad
								# - checking overflow
								# - lr_scheduler step (only if engine.lr_scheduler is not None)
								self.engine.step()
								# and this plugin overrides the above calls with no-ops when Accelerate runs under
								# Deepspeed, but allows normal functionality for non-Deepspeed cases thus enabling a simple
								# training loop that works transparently under many training regimes.







class        lowerCAmelCase__      (   __lowercase	):

						def __init__(			self       ,							a				)					->      Dict:


								'''simple docstring'''

								super().__init__(a       ,							device_placement=a       ,							scaler=a				)
								_UpperCamelCase						    =	hasattr(self.optimizer       ,							"""overflow"""				)

						def  A_				(			self       ,							a=None				)					->      Tuple:


								'''simple docstring'''

								pass  # `accelerator.backward(loss)` is doing that automatically. Therefore, its implementation is not needed

						def  A_				(			self				)					->      Optional[int]:


								'''simple docstring'''

								pass  # `accelerator.backward(loss)` is doing that automatically. Therefore, its implementation is not needed







						@property
						def  A_				(			self				)					->      Any:


								'''simple docstring'''

								if self.__has_overflow__:
										return self.optimizer.overflow
								return False







class        lowerCAmelCase__      (   __lowercase	):

						def __init__(			self       ,							a       ,							a				)					->      int:


								'''simple docstring'''

								super().__init__(a       ,							a				)







						def  A_				(			self				)					->      List[Any]:


								'''simple docstring'''

								pass  # `accelerator.backward(loss)` is doing that automatically. Therefore, its implementation is not needed







class        lowerCAmelCase__      :

						def __init__(			self       ,							a       ,							a=0.001       ,							a=0       ,							**a				)					->      Optional[int]:


								'''simple docstring'''

								_UpperCamelCase						    =	params
								_UpperCamelCase						    =	lr
								_UpperCamelCase						    =	weight_decay
								_UpperCamelCase						    =	kwargs







class        lowerCAmelCase__      :

						def __init__(			self       ,							a       ,							a=None       ,							a=0       ,							**a				)					->      Any:


								'''simple docstring'''

								_UpperCamelCase						    =	optimizer
								_UpperCamelCase						    =	total_num_steps
								_UpperCamelCase						    =	warmup_num_steps
								_UpperCamelCase						    =	kwargs





import torch
import torch.nn as nn
from transformers import CLIPConfig, CLIPVisionModel, PreTrainedModel

from ...utils import logging


lowerCamelCase__	=							logging.get_logger(__name__)


def 							__A(lowerCAmelCase						,  lowerCAmelCase )		->				List[str]:

		"""simple docstring"""






		_UpperCamelCase						    =	nn.functional.normalize(lowerCAmelCase )
		_UpperCamelCase						    =	nn.functional.normalize(lowerCAmelCase )
		return torch.mm(lowerCAmelCase						,  normalized_text_embeds.t() )







class        lowerCAmelCase__      (   __lowercase	):
						UpperCamelCase_							: Tuple	=		CLIPConfig

						UpperCamelCase_							: str	=		["CLIPEncoderLayer"]

						def __init__(			self       ,							a				)					->      Optional[Any]:


								'''simple docstring'''

								super().__init__(a				)

								_UpperCamelCase						    =	CLIPVisionModel(config.vision_config				)
								_UpperCamelCase						    =	nn.Linear(config.vision_config.hidden_size       ,							config.projection_dim       ,							bias=a				)

								_UpperCamelCase						    =	nn.Parameter(torch.ones(17       ,							config.projection_dim				)       ,							requires_grad=a				)
								_UpperCamelCase						    =	nn.Parameter(torch.ones(3       ,							config.projection_dim				)       ,							requires_grad=a				)

								_UpperCamelCase						    =	nn.Parameter(torch.ones(17				)       ,							requires_grad=a				)
								_UpperCamelCase						    =	nn.Parameter(torch.ones(3				)       ,							requires_grad=a				)

						@torch.no_grad()
						def  A_				(			self       ,							a       ,							a				)					->      str:


								'''simple docstring'''

								_UpperCamelCase						    =	self.vision_model(a				)[1]  # pooled_output
								_UpperCamelCase						    =	self.visual_projection(a				)

								# we always cast to float32 as this does not cause significant overhead and is compatible with bfloat16
								_UpperCamelCase						    =	cosine_distance(a       ,							self.special_care_embeds				).cpu().float().numpy()
								_UpperCamelCase						    =	cosine_distance(a       ,							self.concept_embeds				).cpu().float().numpy()

								_UpperCamelCase						    =	[]
								_UpperCamelCase						    =	image_embeds.shape[0]
								for i in range(a				):
										_UpperCamelCase						    =	{"""special_scores""": {}, """special_care""": [], """concept_scores""": {}, """bad_concepts""": []}

										# increase this value to create a stronger `nfsw` filter
										# at the cost of increasing the possibility of filtering benign images
										_UpperCamelCase						    =	0.0

										for concept_idx in range(len(special_cos_dist[0]				)				):
												_UpperCamelCase						    =	special_cos_dist[i][concept_idx]
												_UpperCamelCase						    =	self.special_care_embeds_weights[concept_idx].item()
												_UpperCamelCase						    =	round(concept_cos - concept_threshold + adjustment       ,							3				)
												if result_img["special_scores"][concept_idx] > 0:
														result_img["special_care"].append({concept_idx, result_img["""special_scores"""][concept_idx]}				)
														_UpperCamelCase						    =	0.01

										for concept_idx in range(len(cos_dist[0]				)				):
												_UpperCamelCase						    =	cos_dist[i][concept_idx]
												_UpperCamelCase						    =	self.concept_embeds_weights[concept_idx].item()
												_UpperCamelCase						    =	round(concept_cos - concept_threshold + adjustment       ,							3				)
												if result_img["concept_scores"][concept_idx] > 0:
														result_img["bad_concepts"].append(a				)

										result.append(a				)

								_UpperCamelCase						    =	[len(res["""bad_concepts"""]				) > 0 for res in result]

								return images, has_nsfw_concepts







						@torch.no_grad()
						def  A_				(			self       ,							a       ,							a				)					->      Any:


								'''simple docstring'''

								_UpperCamelCase						    =	self.vision_model(a				)[1]  # pooled_output
								_UpperCamelCase						    =	self.visual_projection(a				)

								_UpperCamelCase						    =	cosine_distance(a       ,							self.special_care_embeds				)
								_UpperCamelCase						    =	cosine_distance(a       ,							self.concept_embeds				)

								# increase this value to create a stronger `nsfw` filter
								# at the cost of increasing the possibility of filtering benign images
								_UpperCamelCase						    =	0.0

								_UpperCamelCase						    =	special_cos_dist - self.special_care_embeds_weights + adjustment
								# special_scores = special_scores.round(decimals=3)
								_UpperCamelCase						    =	torch.any(special_scores > 0       ,							dim=1				)
								_UpperCamelCase						    =	special_care * 0.01
								_UpperCamelCase						    =	special_adjustment.unsqueeze(1				).expand(-1       ,							cos_dist.shape[1]				)

								_UpperCamelCase						    =	(cos_dist - self.concept_embeds_weights) + special_adjustment
								# concept_scores = concept_scores.round(decimals=3)
								_UpperCamelCase						    =	torch.any(concept_scores > 0       ,							dim=1				)

								return images, has_nsfw_concepts



