

from ...configuration_utils import PretrainedConfig
from ...utils import logging


_UpperCAmelCase						:  List[Any]									=			logging.get_logger(__name__)

_UpperCAmelCase						:  Any									=			{}







class 					lowercase							(			lowercase_     ):
				__SCREAMING_SNAKE_CASE   :					Optional[int]			     =     '''llama'''
				__SCREAMING_SNAKE_CASE   :					Optional[Any]			     =     ['''past_key_values''']




				def __init__(   self ,  snake_case=3_2000 ,  snake_case=4096 ,  snake_case=1_1008 ,  snake_case=32 ,  snake_case=32 ,  snake_case=None ,  snake_case="silu" ,  snake_case=2048 ,  snake_case=0.02 ,  snake_case=1e-6 ,  snake_case=True ,  snake_case=0 ,  snake_case=1 ,  snake_case=2 ,  snake_case=1 ,  snake_case=False ,  snake_case=None ,  **snake_case ,  ):
											snake_case_     							=       vocab_size
											snake_case_     							=       max_position_embeddings
											snake_case_     							=       hidden_size
											snake_case_     							=       intermediate_size
											snake_case_     							=       num_hidden_layers
											snake_case_     							=       num_attention_heads

											# for backward compatibility
											if num_key_value_heads is None:
																		snake_case_     							=       num_attention_heads

											snake_case_     							=       num_key_value_heads
											snake_case_     							=       hidden_act
											snake_case_     							=       initializer_range
											snake_case_     							=       rms_norm_eps
											snake_case_     							=       pretraining_tp
											snake_case_     							=       use_cache
											snake_case_     							=       rope_scaling
											self._rope_scaling_validation()

											super().__init__(
											    pad_token_id=snake_case ,  bos_token_id=snake_case ,  eos_token_id=snake_case ,  tie_word_embeddings=snake_case ,  **snake_case ,  )




				def        a				(   self			):
											if self.rope_scaling is None:
																		return

											if not isinstance(self.rope_scaling ,  snake_case			) or len(self.rope_scaling			) != 2:
																		raise ValueError(
																		    '`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '
																		    F'''got {self.rope_scaling}'''			)
											snake_case_     							=       self.rope_scaling.get('type' ,  snake_case			)
											snake_case_     							=       self.rope_scaling.get('factor' ,  snake_case			)
											if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
																		raise ValueError(
																		    F'''`rope_scaling`\'s name field must be one of [\'linear\', \'dynamic\'], got {rope_scaling_type}'''			)
											if rope_scaling_factor is None or not isinstance(snake_case ,  snake_case			) or rope_scaling_factor <= 1.0:
																		raise ValueError(F'''`rope_scaling`\'s factor field must be an float > 1, got {rope_scaling_factor}'''			)







from typing import TYPE_CHECKING

from ...utils import OptionalDependencyNotAvailable, _LazyModule, is_torch_available


_UpperCAmelCase						:  int									=			{
    """configuration_lilt""": ["""LILT_PRETRAINED_CONFIG_ARCHIVE_MAP""", """LiltConfig"""],
}

try:
					if not is_torch_available():
										raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
					pass
else:
					_UpperCAmelCase						:  str									=			[
					    """LILT_PRETRAINED_MODEL_ARCHIVE_LIST""",
					    """LiltForQuestionAnswering""",
					    """LiltForSequenceClassification""",
					    """LiltForTokenClassification""",
					    """LiltModel""",
					    """LiltPreTrainedModel""",
					]

if TYPE_CHECKING:
					from .configuration_lilt import LILT_PRETRAINED_CONFIG_ARCHIVE_MAP, LiltConfig

					try:
										if not is_torch_available():
															raise OptionalDependencyNotAvailable()
					except OptionalDependencyNotAvailable:
										pass
					else:
										from .modeling_lilt import (
										    LILT_PRETRAINED_MODEL_ARCHIVE_LIST,
										    LiltForQuestionAnswering,
										    LiltForSequenceClassification,
										    LiltForTokenClassification,
										    LiltModel,
										    LiltPreTrainedModel,
										)

else:
					import sys

					_UpperCAmelCase						:  Dict									=			_LazyModule(__name__, globals()["""__file__"""], _import_structure, module_spec=__spec__)




