



'''simple docstring'''

from dataclasses import dataclass, field
from typing import Optional

from transformers import AutoConfig, AutoImageProcessor, AutoTokenizer, FlaxVisionEncoderDecoderModel, HfArgumentParser

@dataclass
class       __magic_name__		:
							lowerCAmelCase			:       str  							=   field(
							    metadata={'help': 'The output directory where the model will be written.'}     ,		)
							lowerCAmelCase			:       str  							=   field(
							    metadata={
							        'help': (
							            'The encoder model checkpoint for weights initialization.'
							            'Don\'t set if you want to train an encoder model from scratch.'
							        )
							    }     ,		)
							lowerCAmelCase			:       str  							=   field(
							    metadata={
							        'help': (
							            'The decoder model checkpoint for weights initialization.'
							            'Don\'t set if you want to train a decoder model from scratch.'
							        )
							    }     ,		)
							lowerCAmelCase			:       Optional[str]  							=   field(
							    default=_UpperCamelCase     ,		metadata={'help': 'Pretrained encoder config name or path if not the same as encoder_model_name'}  )
							lowerCAmelCase			:       Optional[str]  							=   field(
							    default=_UpperCamelCase     ,		metadata={'help': 'Pretrained decoder config name or path if not the same as decoder_model_name'}  )



def 	__lowerCamelCase	( )     -> Dict:
		_a	: Optional[int]								=				HfArgumentParser((ModelArguments,)    )
		((_a)			,   )	: Optional[int]								=				parser.parse_args_into_dataclasses()

		# Load pretrained model and tokenizer

		# Use explicit specified encoder config
		if model_args.encoder_config_name:
				_a	: Any								=				AutoConfig.from_pretrained(model_args.encoder_config_name    )
		# Use pretrained encoder model's config
		else:
				_a	: int								=				AutoConfig.from_pretrained(model_args.encoder_model_name_or_path    )

		# Use explicit specified decoder config
		if model_args.decoder_config_name:
				_a	: List[Any]								=				AutoConfig.from_pretrained(model_args.decoder_config_name    )
		# Use pretrained decoder model's config
		else:
				_a	: Union[str, Any]								=				AutoConfig.from_pretrained(model_args.decoder_model_name_or_path    )

		# necessary for `from_encoder_decoder_pretrained` when `decoder_config` is passed
		_a	: List[Any]								=				True
		_a	: str								=				True

		_a	: Dict								=				FlaxVisionEncoderDecoderModel.from_encoder_decoder_pretrained(
		    encoder_pretrained_model_name_or_path=model_args.encoder_model_name_or_path		,     decoder_pretrained_model_name_or_path=model_args.decoder_model_name_or_path		,     encoder_config=lowerCAmelCase_		,     decoder_config=lowerCAmelCase_		,     )

		# GPT2 only has bos/eos tokens but not decoder_start/pad tokens
		_a	: Union[str, Any]								=				decoder_config.decoder_start_token_id
		_a	: Dict								=				decoder_config.pad_token_id
		if decoder_start_token_id is None:
				_a	: str								=				decoder_config.bos_token_id
		if pad_token_id is None:
				_a	: Optional[Any]								=				decoder_config.eos_token_id

		# This is necessary to make Flax's generate() work
		_a	: Union[str, Any]								=				decoder_config.eos_token_id
		_a	: Optional[Any]								=				decoder_start_token_id
		_a	: List[Any]								=				pad_token_id

		_a	: Any								=				AutoImageProcessor.from_pretrained(model_args.encoder_model_name_or_path    )

		_a	: List[str]								=				AutoTokenizer.from_pretrained(model_args.decoder_model_name_or_path    )
		_a	: Optional[Any]								=				tokenizer.convert_ids_to_tokens(model.config.pad_token_id    )

		model.save_pretrained(model_args.output_dir    )
		image_processor.save_pretrained(model_args.output_dir    )
		tokenizer.save_pretrained(model_args.output_dir    )


if __name__ == "__main__":
							main()









'''simple docstring'''

from ...configuration_utils import PretrainedConfig
from ...utils import logging


__lowerCAmelCase		     =  logging.get_logger(__name__)

__lowerCAmelCase		     =  {
    '''s-JoL/Open-Llama-V1''': '''https://huggingface.co/s-JoL/Open-Llama-V1/blob/main/config.json''',
}

class       __magic_name__		(       _UpperCamelCase  ):
							lowerCAmelCase			:       str  							=   'open-llama'







							def __init__(				self   :			str      ,_UpperCAmelCase   :			int=100000      ,_UpperCAmelCase   :			List[str]=4096      ,_UpperCAmelCase   :			Dict=11008      ,_UpperCAmelCase   :			int=32      ,_UpperCAmelCase   :			Union[str, Any]=32      ,_UpperCAmelCase   :			List[str]="silu"      ,_UpperCAmelCase   :			List[Any]=2048      ,_UpperCAmelCase   :			Any=0.02      ,_UpperCAmelCase   :			int=1E-6      ,_UpperCAmelCase   :			Optional[int]=True      ,_UpperCAmelCase   :			Dict=0      ,_UpperCAmelCase   :			Optional[Any]=1      ,_UpperCAmelCase   :			Dict=2      ,_UpperCAmelCase   :			Tuple=False      ,_UpperCAmelCase   :			Dict=True      ,_UpperCAmelCase   :			Dict=0.1      ,_UpperCAmelCase   :			Optional[int]=0.1      ,_UpperCAmelCase   :			List[str]=True      ,_UpperCAmelCase   :			int=True      ,_UpperCAmelCase   :			Union[str, Any]=None      ,**_UpperCAmelCase   :			Optional[int]      ,):
									_a	: str								=				vocab_size
									_a	: str								=				max_position_embeddings
									_a	: List[str]								=				hidden_size
									_a	: Any								=				intermediate_size
									_a	: Union[str, Any]								=				num_hidden_layers
									_a	: Tuple								=				num_attention_heads
									_a	: int								=				hidden_act
									_a	: str								=				initializer_range
									_a	: Any								=				rms_norm_eps
									_a	: Dict								=				use_cache
									_a	: Optional[int]								=				kwargs.pop(
									    'use_memorry_efficient_attention'      ,_UpperCAmelCase						)
									_a	: int								=				hidden_dropout_prob
									_a	: int								=				attention_dropout_prob
									_a	: Union[str, Any]								=				use_stable_embedding
									_a	: str								=				shared_input_output_embedding
									_a	: Any								=				rope_scaling
									self._rope_scaling_validation()

									super().__init__(
									    pad_token_id=_UpperCAmelCase      ,bos_token_id=_UpperCAmelCase      ,eos_token_id=_UpperCAmelCase      ,tie_word_embeddings=_UpperCAmelCase      ,**_UpperCAmelCase      ,)







							def 				__lowercase     (				self   :			Optional[Any]						):
									if self.rope_scaling is None:
											return

									if not isinstance(self.rope_scaling      ,_UpperCAmelCase						) or len(self.rope_scaling						) != 2:
											raise ValueError(
											    '`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '
											    F"""got {self.rope_scaling}"""						)
									_a	: Optional[Any]								=				self.rope_scaling.get('type'      ,_UpperCAmelCase						)
									_a	: Optional[Any]								=				self.rope_scaling.get('factor'      ,_UpperCAmelCase						)
									if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
											raise ValueError(
											    F"""`rope_scaling`'s name field must be one of ['linear', 'dynamic'], got {rope_scaling_type}"""						)
									if rope_scaling_factor is None or not isinstance(_UpperCAmelCase      ,_UpperCAmelCase						) or rope_scaling_factor <= 1.0:
											raise ValueError(F"""`rope_scaling`'s factor field must be an float > 1, got {rope_scaling_factor}"""						)




