





import math
from enum import Enum
from typing import Optional, Union

from torch.optim import Optimizer
from torch.optim.lr_scheduler import LambdaLR

from .utils import logging


UpperCamelCase__      =logging.get_logger(__name__)





class 			lowerCAmelCase__( __lowercase    ):






       '''simple docstring'''


       __snake_case 			=    'linear'
       __snake_case 			=    'cosine'
       __snake_case 			=    'cosine_with_restarts'
       __snake_case 			=    'polynomial'
       __snake_case 			=    'constant'
       __snake_case 			=    'constant_with_warmup'
       __snake_case 			=    'piecewise_constant'






def     lowerCamelCase__			(__lowerCamelCase,       __lowerCamelCase = -1	):
     return LambdaLR(__lowerCamelCase,       lambda __lowerCamelCase	: 1,       last_epoch=__lowerCamelCase	)






def     lowerCamelCase__			(__lowerCamelCase,       __lowerCamelCase,       __lowerCamelCase = -1	):

     def lr_lambda(__lowerCamelCase	):
          if current_step < num_warmup_steps:
               return float(__lowerCamelCase	) / float(max(1.0,       __lowerCamelCase	)	)
          return 1.0

     return LambdaLR(__lowerCamelCase,       __lowerCamelCase,       last_epoch=__lowerCamelCase	)






def     lowerCamelCase__			(__lowerCamelCase,       __lowerCamelCase,       __lowerCamelCase = -1	):
     _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							{}
     _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							step_rules.split(","	)
     for rule_str in rule_list[:-1]:
          _SCREAMING_SNAKE_CASE    ,			_SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							rule_str.split(":"	)
          _SCREAMING_SNAKE_CASE       :	str						    =							int(__lowerCamelCase	)
          _SCREAMING_SNAKE_CASE       :	Tuple						    =							float(__lowerCamelCase	)
          _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							value
     _SCREAMING_SNAKE_CASE       :	List[str]						    =							float(rule_list[-1]	)

     def create_rules_function(__lowerCamelCase,       __lowerCamelCase	):
          def rule_func(__lowerCamelCase	) -> float:
               _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							sorted(rules_dict.keys()	)
               for i, sorted_step in enumerate(__lowerCamelCase	):
                    if steps < sorted_step:
                         return rules_dict[sorted_steps[i]]
               return last_lr_multiple

          return rule_func

     _SCREAMING_SNAKE_CASE       :	List[Any]						    =							create_rules_function(__lowerCamelCase,       __lowerCamelCase	)

     return LambdaLR(__lowerCamelCase,       __lowerCamelCase,       last_epoch=__lowerCamelCase	)






def     lowerCamelCase__			(__lowerCamelCase,       __lowerCamelCase,       __lowerCamelCase,       __lowerCamelCase=-1	):

     def lr_lambda(__lowerCamelCase	):
          if current_step < num_warmup_steps:
               return float(__lowerCamelCase	) / float(max(1,       __lowerCamelCase	)	)
          return max(
              0.0,       float(num_training_steps - current_step	) / float(max(1,       num_training_steps - num_warmup_steps	)	)	)

     return LambdaLR(__lowerCamelCase,       __lowerCamelCase,       __lowerCamelCase	)






def     lowerCamelCase__			(__lowerCamelCase,       __lowerCamelCase,       __lowerCamelCase,       __lowerCamelCase = 0.5,       __lowerCamelCase = -1	):

     def lr_lambda(__lowerCamelCase	):
          if current_step < num_warmup_steps:
               return float(__lowerCamelCase	) / float(max(1,       __lowerCamelCase	)	)
          _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							float(current_step - num_warmup_steps	) / float(max(1,       num_training_steps - num_warmup_steps	)	)
          return max(0.0,       0.5 * (1.0 + math.cos(math.pi * float(__lowerCamelCase	) * 2.0 * progress	))	)

     return LambdaLR(__lowerCamelCase,       __lowerCamelCase,       __lowerCamelCase	)






def     lowerCamelCase__			(__lowerCamelCase,       __lowerCamelCase,       __lowerCamelCase,       __lowerCamelCase = 1,       __lowerCamelCase = -1	):

     def lr_lambda(__lowerCamelCase	):
          if current_step < num_warmup_steps:
               return float(__lowerCamelCase	) / float(max(1,       __lowerCamelCase	)	)
          _SCREAMING_SNAKE_CASE       :	Any						    =							float(current_step - num_warmup_steps	) / float(max(1,       num_training_steps - num_warmup_steps	)	)
          if progress >= 1.0:
               return 0.0
          return max(0.0,       0.5 * (1.0 + math.cos(math.pi * ((float(__lowerCamelCase	) * progress) % 1.0)	))	)

     return LambdaLR(__lowerCamelCase,       __lowerCamelCase,       __lowerCamelCase	)






def     lowerCamelCase__			(__lowerCamelCase,       __lowerCamelCase,       __lowerCamelCase,       __lowerCamelCase=1e-7,       __lowerCamelCase=1.0,       __lowerCamelCase=-1	):
     _SCREAMING_SNAKE_CASE       :	str						    =							optimizer.defaults["lr"]
     if not (lr_init > lr_end):
          raise ValueError(f"""lr_end ({lr_end}) must be be smaller than initial lr ({lr_init})"""	)

     def lr_lambda(__lowerCamelCase	):
          if current_step < num_warmup_steps:
               return float(__lowerCamelCase	) / float(max(1,       __lowerCamelCase	)	)
          elif current_step > num_training_steps:
               return lr_end / lr_init  # as LambdaLR multiplies by lr_init
          else:
               _SCREAMING_SNAKE_CASE       :	List[Any]						    =							lr_init - lr_end
               _SCREAMING_SNAKE_CASE       :	List[Any]						    =							num_training_steps - num_warmup_steps
               _SCREAMING_SNAKE_CASE       :	List[Any]						    =							1 - (current_step - num_warmup_steps) / decay_steps
               _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							lr_range * pct_remaining**power + lr_end
               return decay / lr_init  # as LambdaLR multiplies by lr_init

     return LambdaLR(__lowerCamelCase,       __lowerCamelCase,       __lowerCamelCase	)


UpperCamelCase__      ={
    SchedulerType.LINEAR: get_linear_schedule_with_warmup,
    SchedulerType.COSINE: get_cosine_schedule_with_warmup,
    SchedulerType.COSINE_WITH_RESTARTS: get_cosine_with_hard_restarts_schedule_with_warmup,
    SchedulerType.POLYNOMIAL: get_polynomial_decay_schedule_with_warmup,
    SchedulerType.CONSTANT: get_constant_schedule,
    SchedulerType.CONSTANT_WITH_WARMUP: get_constant_schedule_with_warmup,
    SchedulerType.PIECEWISE_CONSTANT: get_piecewise_constant_schedule,
}






def     lowerCamelCase__			(__lowerCamelCase,       __lowerCamelCase,       __lowerCamelCase = None,       __lowerCamelCase = None,       __lowerCamelCase = None,       __lowerCamelCase = 1,       __lowerCamelCase = 1.0,       __lowerCamelCase = -1,       ):
     _SCREAMING_SNAKE_CASE       :	List[str]						    =							SchedulerType(__lowerCamelCase	)
     _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							TYPE_TO_SCHEDULER_FUNCTION[name]
     if name == SchedulerType.CONSTANT:
          return schedule_func(__lowerCamelCase,       last_epoch=__lowerCamelCase	)

     if name == SchedulerType.PIECEWISE_CONSTANT:
          return schedule_func(__lowerCamelCase,       step_rules=__lowerCamelCase,       last_epoch=__lowerCamelCase	)

     # All other schedulers require `num_warmup_steps`
     if num_warmup_steps is None:
          raise ValueError(f"""{name} requires `num_warmup_steps`, please provide that argument."""	)

     if name == SchedulerType.CONSTANT_WITH_WARMUP:
          return schedule_func(__lowerCamelCase,       num_warmup_steps=__lowerCamelCase,       last_epoch=__lowerCamelCase	)

     # All other schedulers require `num_training_steps`
     if num_training_steps is None:
          raise ValueError(f"""{name} requires `num_training_steps`, please provide that argument."""	)

     if name == SchedulerType.COSINE_WITH_RESTARTS:
          return schedule_func(
              __lowerCamelCase,       num_warmup_steps=__lowerCamelCase,       num_training_steps=__lowerCamelCase,       num_cycles=__lowerCamelCase,       last_epoch=__lowerCamelCase,       )

     if name == SchedulerType.POLYNOMIAL:
          return schedule_func(
              __lowerCamelCase,       num_warmup_steps=__lowerCamelCase,       num_training_steps=__lowerCamelCase,       power=__lowerCamelCase,       last_epoch=__lowerCamelCase,       )

     return schedule_func(
         __lowerCamelCase,       num_warmup_steps=__lowerCamelCase,       num_training_steps=__lowerCamelCase,       last_epoch=__lowerCamelCase	)






import copy
import os
from typing import TYPE_CHECKING, List, Union


if TYPE_CHECKING:
       pass

from ...configuration_utils import PretrainedConfig
from ...utils import logging


UpperCamelCase__      =logging.get_logger(__name__)

UpperCamelCase__      ={
    'kakaobrain/align-base': 'https://huggingface.co/kakaobrain/align-base/resolve/main/config.json',
}





class 			lowerCAmelCase__( __lowercase    ):






       '''simple docstring'''


       __snake_case 			=    'align_text_model'




       def __init__(      self    ,						__lowerCamelCase=3_0_5_2_2    ,						__lowerCamelCase=7_6_8    ,						__lowerCamelCase=1_2    ,						__lowerCamelCase=1_2    ,						__lowerCamelCase=3_0_7_2    ,						__lowerCamelCase="gelu"    ,						__lowerCamelCase=0.1    ,						__lowerCamelCase=0.1    ,						__lowerCamelCase=5_1_2    ,						__lowerCamelCase=2    ,						__lowerCamelCase=0.02    ,						__lowerCamelCase=1E-12    ,						__lowerCamelCase=0    ,						__lowerCamelCase="absolute"    ,						__lowerCamelCase=True    ,						**__lowerCamelCase    ,						)			->     List[Any]:
            super().__init__(**__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE       :	Tuple						    =							vocab_size
            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							hidden_size
            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							num_hidden_layers
            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							num_attention_heads
            _SCREAMING_SNAKE_CASE       :	int						    =							hidden_act
            _SCREAMING_SNAKE_CASE       :	Any						    =							intermediate_size
            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							hidden_dropout_prob
            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							attention_probs_dropout_prob
            _SCREAMING_SNAKE_CASE       :	Dict						    =							max_position_embeddings
            _SCREAMING_SNAKE_CASE       :	Optional[Any]						    =							type_vocab_size
            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							initializer_range
            _SCREAMING_SNAKE_CASE       :	Dict						    =							layer_norm_eps
            _SCREAMING_SNAKE_CASE       :	str						    =							position_embedding_type
            _SCREAMING_SNAKE_CASE       :	Dict						    =							use_cache
            _SCREAMING_SNAKE_CASE       :	List[str]						    =							pad_token_id







       @classmethod
       def     UpperCamelCase_			(      cls    ,						__lowerCamelCase    ,						**__lowerCamelCase							)			->     "PretrainedConfig":
            cls._set_token_in_kwargs(__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE    ,			_SCREAMING_SNAKE_CASE       :	List[Any]						    =							cls.get_config_dict(__lowerCamelCase    ,						**__lowerCamelCase							)

            # get the text config dict if we are loading from AlignConfig
            if config_dict.get("model_type"							) == "align":
                 _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							config_dict["text_config"]

            if "model_type" in config_dict and hasattr(cls    ,						"model_type"							) and config_dict["model_type"] != cls.model_type:
                 logger.warning(
                     F"""You are using a model of type {config_dict['model_type']} to instantiate a model of type """
                     F"""{cls.model_type}. This is not supported for all configurations of models and can yield errors."""							)

            return cls.from_dict(__lowerCamelCase    ,						**__lowerCamelCase							)





class 			lowerCAmelCase__( __lowercase    ):






       '''simple docstring'''


       __snake_case 			=    'align_vision_model'




       def __init__(      self    ,						__lowerCamelCase = 3    ,						__lowerCamelCase = 6_0_0    ,						__lowerCamelCase = 2.0    ,						__lowerCamelCase = 3.1    ,						__lowerCamelCase = 8    ,						__lowerCamelCase = [3, 3, 5, 3, 5, 5, 3]    ,						__lowerCamelCase = [3_2, 1_6, 2_4, 4_0, 8_0, 1_1_2, 1_9_2]    ,						__lowerCamelCase = [1_6, 2_4, 4_0, 8_0, 1_1_2, 1_9_2, 3_2_0]    ,						__lowerCamelCase = []    ,						__lowerCamelCase = [1, 2, 2, 2, 1, 2, 1]    ,						__lowerCamelCase = [1, 2, 2, 3, 3, 4, 1]    ,						__lowerCamelCase = [1, 6, 6, 6, 6, 6, 6]    ,						__lowerCamelCase = 0.25    ,						__lowerCamelCase = "swish"    ,						__lowerCamelCase = 2_5_6_0    ,						__lowerCamelCase = "mean"    ,						__lowerCamelCase = 0.02    ,						__lowerCamelCase = 0.001    ,						__lowerCamelCase = 0.99    ,						__lowerCamelCase = 0.2    ,						**__lowerCamelCase    ,						)			->     Dict:
            super().__init__(**__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE       :	int						    =							num_channels
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							image_size
            _SCREAMING_SNAKE_CASE       :	Dict						    =							width_coefficient
            _SCREAMING_SNAKE_CASE       :	str						    =							depth_coefficient
            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							depth_divisor
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							kernel_sizes
            _SCREAMING_SNAKE_CASE       :	Tuple						    =							in_channels
            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							out_channels
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							depthwise_padding
            _SCREAMING_SNAKE_CASE       :	str						    =							strides
            _SCREAMING_SNAKE_CASE       :	List[str]						    =							num_block_repeats
            _SCREAMING_SNAKE_CASE       :	Tuple						    =							expand_ratios
            _SCREAMING_SNAKE_CASE       :	int						    =							squeeze_expansion_ratio
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							hidden_act
            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							hidden_dim
            _SCREAMING_SNAKE_CASE       :	Dict						    =							pooling_type
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							initializer_range
            _SCREAMING_SNAKE_CASE       :	List[Any]						    =							batch_norm_eps
            _SCREAMING_SNAKE_CASE       :	Union[str, Any]						    =							batch_norm_momentum
            _SCREAMING_SNAKE_CASE       :	int						    =							drop_connect_rate
            _SCREAMING_SNAKE_CASE       :	Tuple						    =							sum(__lowerCamelCase							) * 4







       @classmethod
       def     UpperCamelCase_			(      cls    ,						__lowerCamelCase    ,						**__lowerCamelCase							)			->     "PretrainedConfig":
            cls._set_token_in_kwargs(__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE    ,			_SCREAMING_SNAKE_CASE       :	int						    =							cls.get_config_dict(__lowerCamelCase    ,						**__lowerCamelCase							)

            # get the vision config dict if we are loading from AlignConfig
            if config_dict.get("model_type"							) == "align":
                 _SCREAMING_SNAKE_CASE       :	int						    =							config_dict["vision_config"]

            if "model_type" in config_dict and hasattr(cls    ,						"model_type"							) and config_dict["model_type"] != cls.model_type:
                 logger.warning(
                     F"""You are using a model of type {config_dict['model_type']} to instantiate a model of type """
                     F"""{cls.model_type}. This is not supported for all configurations of models and can yield errors."""							)

            return cls.from_dict(__lowerCamelCase    ,						**__lowerCamelCase							)





class 			lowerCAmelCase__( __lowercase    ):






       '''simple docstring'''


       __snake_case 			=    'align'
       __snake_case 			=    True




       def __init__(      self    ,						__lowerCamelCase=None    ,						__lowerCamelCase=None    ,						__lowerCamelCase=6_4_0    ,						__lowerCamelCase=1.0    ,						__lowerCamelCase=0.02    ,						**__lowerCamelCase    ,						)			->     List[Any]:
            super().__init__(**__lowerCamelCase							)

            if text_config is None:
                 _SCREAMING_SNAKE_CASE       :	List[Any]						    =							{}
                 logger.info("text_config is None. Initializing the AlignTextConfig with default values."							)

            if vision_config is None:
                 _SCREAMING_SNAKE_CASE       :	List[str]						    =							{}
                 logger.info("vision_config is None. Initializing the AlignVisionConfig with default values."							)

            _SCREAMING_SNAKE_CASE       :	Dict						    =							AlignTextConfig(**__lowerCamelCase							)
            _SCREAMING_SNAKE_CASE       :	Any						    =							AlignVisionConfig(**__lowerCamelCase							)

            _SCREAMING_SNAKE_CASE       :	int						    =							projection_dim
            _SCREAMING_SNAKE_CASE       :	List[str]						    =							temperature_init_value
            _SCREAMING_SNAKE_CASE       :	Any						    =							initializer_range




       @classmethod
       def     UpperCamelCase_			(      cls    ,						__lowerCamelCase    ,						__lowerCamelCase    ,						**__lowerCamelCase							)			->     List[str]:
            return cls(text_config=text_config.to_dict()    ,						vision_config=vision_config.to_dict()    ,						**__lowerCamelCase							)







       def     UpperCamelCase_			(      self							)			->     Union[str, Any]:
            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							copy.deepcopy(self.__dict__							)
            _SCREAMING_SNAKE_CASE       :	Any						    =							self.text_config.to_dict()
            _SCREAMING_SNAKE_CASE       :	Optional[int]						    =							self.vision_config.to_dict()
            _SCREAMING_SNAKE_CASE       :	Dict						    =							self.__class__.model_type
            return output