





import collections
import importlib.util
import os
import re
from pathlib import Path


__UpperCamelCase:	List[Any] = """src/transformers"""


# Matches is_xxx_available()
__UpperCamelCase:	Tuple = re.compile(r"""is\_([a-z_]*)_available()""")
# Catches a one-line _import_struct = {xxx}
__UpperCamelCase:	Optional[int] = re.compile(r"""^_import_structure\s+=\s+\{([^\}]+)\}""")
# Catches a line with a key-values pattern: "bla": ["foo", "bar"]
__UpperCamelCase:	List[Any] = re.compile(r"""\s+\"\S*\":\s+\[([^\]]*)\]""")
# Catches a line if not is_foo_available
__UpperCamelCase:	Any = re.compile(r"""^\s*if\s+not\s+is\_[a-z_]*\_available\(\)""")
# Catches a line _import_struct["bla"].append("foo")
__UpperCamelCase:	Union[str, Any] = re.compile(r"""^\s*_import_structure\[\"\S*\"\]\.append\(\"(\S*)\"\)""")
# Catches a line _import_struct["bla"].extend(["foo", "bar"]) or _import_struct["bla"] = ["foo", "bar"]
__UpperCamelCase:	Optional[Any] = re.compile(r"""^\s*_import_structure\[\S*\](?:\.extend\(|\s*=\s+)\[([^\]]*)\]""")
# Catches a line with an object between quotes and a comma:     "MyModel",
__UpperCamelCase:	Any = re.compile("""^\s+\"([^\"]+)\",""")
# Catches a line with objects between brackets only:    ["foo", "bar"],
__UpperCamelCase:	List[Any] = re.compile("""^\s+\[([^\]]+)\]""")
# Catches a line with from foo import bar, bla, boo
__UpperCamelCase:	Optional[Any] = re.compile(r"""\s+from\s+\S*\s+import\s+([^\(\s].*)\n""")
# Catches a line with try:
__UpperCamelCase:	Any = re.compile(r"""^\s*try:""")
# Catches a line with else:
__UpperCamelCase:	Union[str, Any] = re.compile(r"""^\s*else:""")


def       SCREAMING_SNAKE_CASE__  (       _lowercase     : Tuple			)   ->							Any:




		'''simple docstring'''
		if _re_test_backend.search(_lowercase			) is None:
				return None
		lowercase__					:      Tuple      =		[b[0] for b in _re_backend.findall(_lowercase			)]
		backends.sort()
		return "_and_".join(_lowercase			)


def       SCREAMING_SNAKE_CASE__  (       _lowercase     : List[Any]			)   ->							Optional[Any]:




		'''simple docstring'''
		with open(_lowercase   ,    'r'   ,    encoding='utf-8'   ,    newline='\n'			) as f:
				lowercase__					:      List[str]      =		f.readlines()

		lowercase__					:      str      =		0
		while line_index < len(_lowercase			) and not lines[line_index].startswith('_import_structure = {'			):
				line_index += 1

		# If this is a traditional init, just return.
		if line_index >= len(_lowercase			):
				return None

		# First grab the objects without a specific backend in _import_structure
		lowercase__					:      Dict      =		[]
		while not lines[line_index].startswith('if TYPE_CHECKING'			) and find_backend(lines[line_index]			) is None:
				lowercase__					:      List[Any]      =		lines[line_index]
				# If we have everything on a single line, let's deal with it.
				if _re_one_line_import_struct.search(_lowercase			):
						lowercase__					:      Optional[int]      =		_re_one_line_import_struct.search(_lowercase			).groups()[0]
						lowercase__					:      Tuple      =		re.findall('\[([^\]]+)\]'   ,    _lowercase			)
						for imp in imports:
								objects.extend([obj[1:-1] for obj in imp.split(', '			)]			)
						line_index += 1
						continue
				lowercase__					:      List[str]      =		_re_import_struct_key_value.search(_lowercase			)
				if single_line_import_search is not None:
						lowercase__					:      Optional[int]      =		[obj[1:-1] for obj in single_line_import_search.groups()[0].split(', '			) if len(_lowercase			) > 0]
						objects.extend(_lowercase			)
				elif line.startswith(' ' * 8 + '"'			):
						objects.append(line[9:-3]			)
				line_index += 1

		lowercase__					:      Dict      =		{'none': objects}
		# Let's continue with backend-specific objects in _import_structure
		while not lines[line_index].startswith('if TYPE_CHECKING'			):
				# If the line is an if not is_backend_available, we grab all objects associated.
				lowercase__					:      Dict      =		find_backend(lines[line_index]			)
				# Check if the backend declaration is inside a try block:
				if _re_try.search(lines[line_index - 1]			) is None:
						lowercase__					:      str      =		None

				if backend is not None:
						line_index += 1

						# Scroll until we hit the else block of try-except-else
						while _re_else.search(lines[line_index]			) is None:
								line_index += 1

						line_index += 1

						lowercase__					:      Dict      =		[]
						# Until we unindent, add backend objects to the list
						while len(lines[line_index]			) <= 1 or lines[line_index].startswith(' ' * 4			):
								lowercase__					:      Optional[int]      =		lines[line_index]
								if _re_import_struct_add_one.search(_lowercase			) is not None:
										objects.append(_re_import_struct_add_one.search(_lowercase			).groups()[0]			)
								elif _re_import_struct_add_many.search(_lowercase			) is not None:
										lowercase__					:      Union[str, Any]      =		_re_import_struct_add_many.search(_lowercase			).groups()[0].split(', '			)
										lowercase__					:      Union[str, Any]      =		[obj[1:-1] for obj in imports if len(_lowercase			) > 0]
										objects.extend(_lowercase			)
								elif _re_between_brackets.search(_lowercase			) is not None:
										lowercase__					:      int      =		_re_between_brackets.search(_lowercase			).groups()[0].split(', '			)
										lowercase__					:      str      =		[obj[1:-1] for obj in imports if len(_lowercase			) > 0]
										objects.extend(_lowercase			)
								elif _re_quote_object.search(_lowercase			) is not None:
										objects.append(_re_quote_object.search(_lowercase			).groups()[0]			)
								elif line.startswith(' ' * 8 + '"'			):
										objects.append(line[9:-3]			)
								elif line.startswith(' ' * 12 + '"'			):
										objects.append(line[13:-3]			)
								line_index += 1

						lowercase__					:      Optional[Any]      =		objects
				else:
						line_index += 1

    # At this stage we are in the TYPE_CHECKING part, first grab the objects without a specific backend
		lowercase__					:      Tuple      =		[]
		while (
		    line_index < len(_lowercase			)
		    and find_backend(lines[line_index]			) is None
		    and not lines[line_index].startswith('else'			)
		):
				lowercase__					:      Any      =		lines[line_index]
				lowercase__					:      Tuple      =		_re_import.search(_lowercase			)
				if single_line_import_search is not None:
						objects.extend(single_line_import_search.groups()[0].split(', '			)			)
				elif line.startswith(' ' * 8			):
						objects.append(line[8:-2]			)
				line_index += 1

		lowercase__					:      Dict      =		{'none': objects}
		# Let's continue with backend-specific objects
		while line_index < len(_lowercase			):
				# If the line is an if is_backend_available, we grab all objects associated.
				lowercase__					:      List[Any]      =		find_backend(lines[line_index]			)
				# Check if the backend declaration is inside a try block:
				if _re_try.search(lines[line_index - 1]			) is None:
						lowercase__					:      Tuple      =		None

				if backend is not None:
						line_index += 1

						# Scroll until we hit the else block of try-except-else
						while _re_else.search(lines[line_index]			) is None:
								line_index += 1

						line_index += 1

						lowercase__					:      List[Any]      =		[]
						# Until we unindent, add backend objects to the list
						while len(lines[line_index]			) <= 1 or lines[line_index].startswith(' ' * 8			):
								lowercase__					:      Union[str, Any]      =		lines[line_index]
								lowercase__					:      Optional[Any]      =		_re_import.search(_lowercase			)
								if single_line_import_search is not None:
										objects.extend(single_line_import_search.groups()[0].split(', '			)			)
								elif line.startswith(' ' * 12			):
										objects.append(line[12:-2]			)
								line_index += 1

						lowercase__					:      Tuple      =		objects
				else:
						line_index += 1

		return import_dict_objects, type_hint_objects


def       SCREAMING_SNAKE_CASE__  (       _lowercase     : Optional[Any]   ,    _lowercase     : Optional[int]			)   ->							Union[str, Any]:




		'''simple docstring'''

		def find_duplicates(_lowercase     : str			):
				return [k for k, v in collections.Counter(_lowercase			).items() if v > 1]

		if list(import_dict_objects.keys()			) != list(type_hint_objects.keys()			):
				return ["Both sides of the init do not have the same backends!"]

		lowercase__					:      Optional[Any]      =		[]
		for key in import_dict_objects.keys():
				lowercase__					:      str      =		find_duplicates(import_dict_objects[key]			)
				if duplicate_imports:
						errors.append(f"""Duplicate _import_structure definitions for: {duplicate_imports}"""			)
				lowercase__					:      int      =		find_duplicates(type_hint_objects[key]			)
				if duplicate_type_hints:
						errors.append(f"""Duplicate TYPE_CHECKING objects for: {duplicate_type_hints}"""			)

				if sorted(set(import_dict_objects[key]			)			) != sorted(set(type_hint_objects[key]			)			):
						lowercase__					:      Any      =		'base imports' if key == 'none' else f"""{key} backend"""
						errors.append(f"""Differences for {name}:"""			)
						for a in type_hint_objects[key]:
								if a not in import_dict_objects[key]:
										errors.append(f"""  {a} in TYPE_HINT but not in _import_structure."""			)
						for a in import_dict_objects[key]:
								if a not in type_hint_objects[key]:
										errors.append(f"""  {a} in _import_structure but not in TYPE_HINT."""			)
		return errors


def       SCREAMING_SNAKE_CASE__  (       )   ->							Any:




		'''simple docstring'''
		lowercase__					:      str      =		[]
		for root, _, files in os.walk(_lowercase			):
				if "__init__.py" in files:
						lowercase__					:      Union[str, Any]      =		os.path.join(_lowercase   ,    '__init__.py'			)
						lowercase__					:      str      =		parse_init(_lowercase			)
						if objects is not None:
								lowercase__					:      Tuple      =		analyze_results(*_lowercase			)
								if len(_lowercase			) > 0:
										lowercase__					:      Tuple      =		f"""Problem in {fname}, both halves do not define the same objects.\n{errors[0]}"""
										failures.append('\n'.join(_lowercase			)			)
		if len(_lowercase			) > 0:
				raise ValueError('\n\n'.join(_lowercase			)			)


def       SCREAMING_SNAKE_CASE__  (       )   ->							Optional[int]:




		'''simple docstring'''
		lowercase__					:      str      =		[]
		for path, directories, files in os.walk(_lowercase			):
				for folder in directories:
						# Ignore private modules
						if folder.startswith('_'			):
								directories.remove(_lowercase			)
								continue
						# Ignore leftovers from branches (empty folders apart from pycache)
						if len(list((Path(_lowercase			) / folder).glob('*.py'			)			)			) == 0:
								continue
						lowercase__					:      Union[str, Any]      =		str((Path(_lowercase			) / folder).relative_to(_lowercase			)			)
						lowercase__					:      Optional[int]      =		short_path.replace(os.path.sep   ,    '.'			)
						submodules.append(_lowercase			)
				for fname in files:
						if fname == "__init__.py":
								continue
						lowercase__					:      Any      =		str((Path(_lowercase			) / fname).relative_to(_lowercase			)			)
						lowercase__					:      Tuple      =		short_path.replace('.py'   ,    ''			).replace(os.path.sep   ,    '.'			)
						if len(submodule.split('.'			)			) == 1:
								submodules.append(_lowercase			)
		return submodules


__UpperCamelCase:	str = [
    """convert_pytorch_checkpoint_to_tf2""",
    """modeling_flax_pytorch_utils""",
]


def       SCREAMING_SNAKE_CASE__  (       )   ->							Optional[Any]:




		'''simple docstring'''
		# This is to make sure the transformers module imported is the one in the repo.
		lowercase__					:      List[Any]      =		importlib.util.spec_from_file_location(
		    'transformers'   ,    os.path.join(_lowercase   ,    '__init__.py'			)   ,    submodule_search_locations=[PATH_TO_TRANSFORMERS]   ,    )
		lowercase__					:      Optional[int]      =		spec.loader.load_module()

		lowercase__					:      str      =		[
		    module
		    for module in get_transformers_submodules()
		    if module not in IGNORE_SUBMODULES and module not in transformers._import_structure.keys()
		]
		if len(_lowercase			) > 0:
				lowercase__					:      List[Any]      =		'\n'.join(f"""- {module}""" for module in module_not_registered			)
				raise ValueError(
				    'The following submodules are not properly registered in the main init of Transformers:\n'
				    f"""{list_of_modules}\n"""
				    'Make sure they appear somewhere in the keys of `_import_structure` with an empty list as value.'			)


if __name__ == "__main__":
			check_all_inits()
			check_submodules()









import unittest

from parameterized import parameterized

from transformers import OpenLlamaConfig, is_torch_available, set_seed
from transformers.testing_utils import require_torch, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
			import torch

			from transformers import OpenLlamaForCausalLM, OpenLlamaForSequenceClassification, OpenLlamaModel


class       __lowerCAmelCase					:


	'''simple docstring'''





	def __init__(						self:						Any,					lowerCamelCase_:						Optional[Any],					lowerCamelCase_:						List[str]=13,					lowerCamelCase_:						Optional[Any]=7,					lowerCamelCase_:						Optional[Any]=True,					lowerCamelCase_:						Tuple=True,					lowerCamelCase_:						Any=False,					lowerCamelCase_:						Union[str, Any]=True,					lowerCamelCase_:						Optional[Any]=99,					lowerCamelCase_:						Tuple=32,					lowerCamelCase_:						Any=5,					lowerCamelCase_:						Tuple=4,					lowerCamelCase_:						List[Any]=37,					lowerCamelCase_:						Union[str, Any]="gelu",					lowerCamelCase_:						str=0.1,					lowerCamelCase_:						Union[str, Any]=0.1,					lowerCamelCase_:						Any=512,					lowerCamelCase_:						Union[str, Any]=16,					lowerCamelCase_:						Any=2,					lowerCamelCase_:						str=0.0_2,					lowerCamelCase_:						Union[str, Any]=3,					lowerCamelCase_:						List[str]=4,					lowerCamelCase_:						Tuple=None,					):
			lowercase__					:      List[str]      =		parent
			lowercase__					:      str      =		batch_size
			lowercase__					:      Optional[int]      =		seq_length
			lowercase__					:      List[Any]      =		is_training
			lowercase__					:      List[str]      =		use_input_mask
			lowercase__					:      Optional[int]      =		use_token_type_ids
			lowercase__					:      str      =		use_labels
			lowercase__					:      Optional[Any]      =		vocab_size
			lowercase__					:      Dict      =		hidden_size
			lowercase__					:      List[Any]      =		num_hidden_layers
			lowercase__					:      Tuple      =		num_attention_heads
			lowercase__					:      Dict      =		intermediate_size
			lowercase__					:      Tuple      =		hidden_act
			lowercase__					:      List[str]      =		hidden_dropout_prob
			lowercase__					:      List[str]      =		attention_probs_dropout_prob
			lowercase__					:      str      =		max_position_embeddings
			lowercase__					:      List[Any]      =		type_vocab_size
			lowercase__					:      Tuple      =		type_sequence_label_size
			lowercase__					:      Optional[int]      =		initializer_range
			lowercase__					:      str      =		num_labels
			lowercase__					:      Optional[int]      =		num_choices
			lowercase__					:      Dict      =		scope

	def    snake_case__(						self:						Union[str, Any]			):
			lowercase__					:      Any      =		ids_tensor([self.batch_size, self.seq_length],					self.vocab_size			)

			lowercase__					:      List[Any]      =		None
			if self.use_input_mask:
					lowercase__					:      List[str]      =		random_attention_mask([self.batch_size, self.seq_length]			)

			lowercase__					:      Optional[Any]      =		None
			if self.use_token_type_ids:
					lowercase__					:      Tuple      =		ids_tensor([self.batch_size, self.seq_length],					self.type_vocab_size			)

			lowercase__					:      str      =		None
			lowercase__					:      Union[str, Any]      =		None
			lowercase__					:      Tuple      =		None
			if self.use_labels:
					lowercase__					:      Tuple      =		ids_tensor([self.batch_size],					self.type_sequence_label_size			)
					lowercase__					:      Any      =		ids_tensor([self.batch_size, self.seq_length],					self.num_labels			)
					lowercase__					:      int      =		ids_tensor([self.batch_size],					self.num_choices			)

			lowercase__					:      Tuple      =		self.get_config()

			return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels

	def    snake_case__(						self:						Tuple			):
			return OpenLlamaConfig(
			    vocab_size=self.vocab_size,					hidden_size=self.hidden_size,					num_hidden_layers=self.num_hidden_layers,					num_attention_heads=self.num_attention_heads,					intermediate_size=self.intermediate_size,					hidden_act=self.hidden_act,					hidden_dropout_prob=self.hidden_dropout_prob,					attention_probs_dropout_prob=self.attention_probs_dropout_prob,					max_position_embeddings=self.max_position_embeddings,					type_vocab_size=self.type_vocab_size,					is_decoder=lowerCamelCase_,					initializer_range=self.initializer_range,					use_stable_embedding=lowerCamelCase_,					)

	def    snake_case__(						self:						Optional[int],					lowerCamelCase_:						Optional[int],					lowerCamelCase_:						List[Any],					lowerCamelCase_:						List[str],					lowerCamelCase_:						Optional[int],					lowerCamelCase_:						Dict,					lowerCamelCase_:						Optional[int],					lowerCamelCase_:						str			):
			lowercase__					:      Union[str, Any]      =		OpenLlamaModel(config=lowerCamelCase_			)
			model.to(lowerCamelCase_			)
			model.eval()
			lowercase__					:      Union[str, Any]      =		model(lowerCamelCase_,					attention_mask=lowerCamelCase_			)
			lowercase__					:      List[Any]      =		model(lowerCamelCase_			)
			self.parent.assertEqual(result.last_hidden_state.shape,					(self.batch_size, self.seq_length, self.hidden_size)			)

	def    snake_case__(						self:						str,					lowerCamelCase_:						Optional[Any],					lowerCamelCase_:						str,					lowerCamelCase_:						str,					lowerCamelCase_:						Optional[Any],					lowerCamelCase_:						Tuple,					lowerCamelCase_:						Optional[Any],					lowerCamelCase_:						List[str],					lowerCamelCase_:						Dict,					lowerCamelCase_:						Union[str, Any],					):
			lowercase__					:      Tuple      =		True
			lowercase__					:      int      =		OpenLlamaModel(lowerCamelCase_			)
			model.to(lowerCamelCase_			)
			model.eval()
			lowercase__					:      List[Any]      =		model(
			    lowerCamelCase_,					attention_mask=lowerCamelCase_,					encoder_hidden_states=lowerCamelCase_,					encoder_attention_mask=lowerCamelCase_,					)
			lowercase__					:      Optional[int]      =		model(
			    lowerCamelCase_,					attention_mask=lowerCamelCase_,					encoder_hidden_states=lowerCamelCase_,					)
			lowercase__					:      str      =		model(lowerCamelCase_,					attention_mask=lowerCamelCase_			)
			self.parent.assertEqual(result.last_hidden_state.shape,					(self.batch_size, self.seq_length, self.hidden_size)			)

	def    snake_case__(						self:						List[Any],					lowerCamelCase_:						Optional[Any],					lowerCamelCase_:						Any,					lowerCamelCase_:						str,					lowerCamelCase_:						List[str],					lowerCamelCase_:						Any,					lowerCamelCase_:						Dict,					lowerCamelCase_:						int,					lowerCamelCase_:						Any,					lowerCamelCase_:						str,					):
			lowercase__					:      Optional[Any]      =		OpenLlamaForCausalLM(config=lowerCamelCase_			)
			model.to(lowerCamelCase_			)
			model.eval()
			lowercase__					:      List[Any]      =		model(lowerCamelCase_,					attention_mask=lowerCamelCase_,					labels=lowerCamelCase_			)
			self.parent.assertEqual(result.logits.shape,					(self.batch_size, self.seq_length, self.vocab_size)			)

	def    snake_case__(						self:						Tuple,					lowerCamelCase_:						str,					lowerCamelCase_:						Optional[Any],					lowerCamelCase_:						Optional[int],					lowerCamelCase_:						str,					lowerCamelCase_:						List[Any],					lowerCamelCase_:						List[Any],					lowerCamelCase_:						str,					lowerCamelCase_:						Optional[Any],					lowerCamelCase_:						Union[str, Any],					):
			lowercase__					:      Optional[int]      =		True
			lowercase__					:      Optional[int]      =		True
			lowercase__					:      Tuple      =		OpenLlamaForCausalLM(config=lowerCamelCase_			)
			model.to(lowerCamelCase_			)
			model.eval()

			# first forward pass
			lowercase__					:      Any      =		model(
			    lowerCamelCase_,					attention_mask=lowerCamelCase_,					encoder_hidden_states=lowerCamelCase_,					encoder_attention_mask=lowerCamelCase_,					use_cache=lowerCamelCase_,					)
			lowercase__					:      str      =		outputs.past_key_values

			# create hypothetical multiple next token and extent to next_input_ids
			lowercase__					:      Optional[Any]      =		ids_tensor((self.batch_size, 3),					config.vocab_size			)
			lowercase__					:      str      =		ids_tensor((self.batch_size, 3),					vocab_size=2			)

			# append to next input_ids and
			lowercase__					:      Tuple      =		torch.cat([input_ids, next_tokens],					dim=-1			)
			lowercase__					:      List[str]      =		torch.cat([input_mask, next_mask],					dim=-1			)

			lowercase__					:      int      =		model(
			    lowerCamelCase_,					attention_mask=lowerCamelCase_,					encoder_hidden_states=lowerCamelCase_,					encoder_attention_mask=lowerCamelCase_,					output_hidden_states=lowerCamelCase_,					)['hidden_states'][0]
			lowercase__					:      int      =		model(
			    lowerCamelCase_,					attention_mask=lowerCamelCase_,					encoder_hidden_states=lowerCamelCase_,					encoder_attention_mask=lowerCamelCase_,					past_key_values=lowerCamelCase_,					output_hidden_states=lowerCamelCase_,					)['hidden_states'][0]

			# select random slice
			lowercase__					:      Tuple      =		ids_tensor((1,),					output_from_past.shape[-1]			).item()
			lowercase__					:      Union[str, Any]      =		output_from_no_past[:, -3:, random_slice_idx].detach()
			lowercase__					:      Tuple      =		output_from_past[:, :, random_slice_idx].detach()

			self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]			)

			# test that outputs are equal for slice
			self.parent.assertTrue(torch.allclose(lowerCamelCase_,					lowerCamelCase_,					atol=1E-3			)			)





	def    snake_case__(						self:						Optional[Any]			):
			lowercase__					:      Dict      =		self.prepare_config_and_inputs()
			(
			    (
			    lowercase__
			)   ,				(
			    lowercase__
			)   ,				(
			    lowercase__
			)   ,				(
			    lowercase__
			)   ,				(
			    lowercase__
			)   ,				(
			    lowercase__
			)   ,				(
			    lowercase__
			)   ,				
			)					:      List[str]      =		config_and_inputs
			lowercase__					:      Any      =		{'input_ids': input_ids, 'attention_mask': input_mask}
			return config, inputs_dict


@require_torch
class       __lowerCAmelCase					(       _UpperCamelCase   ,    _UpperCamelCase   ,    _UpperCamelCase   ,    unittest.TestCase      ):


	'''simple docstring'''





	_A													=				(
	    (OpenLlamaModel, OpenLlamaForCausalLM, OpenLlamaForSequenceClassification) if is_torch_available() else ()
	)
	_A													=				(OpenLlamaForCausalLM,) if is_torch_available() else ()
	_A													=				(
	    {
	        "feature-extraction": OpenLlamaModel,
	        "text-classification": OpenLlamaForSequenceClassification,
	        "text-generation": OpenLlamaForCausalLM,
	        "zero-shot": OpenLlamaForSequenceClassification,
	    }
	    if is_torch_available()
	    else {}
	)
	_A													=				False
	_A													=				False

	def    snake_case__(						self:						Any			):
			lowercase__					:      List[Any]      =		OpenLlamaModelTester(self			)
			lowercase__					:      Dict      =		ConfigTester(self,					config_class=lowerCamelCase_,					hidden_size=37			)

	def    snake_case__(						self:						Any			):
			self.config_tester.run_common_tests()

	def    snake_case__(						self:						Dict			):
			lowercase__					:      Any      =		self.model_tester.prepare_config_and_inputs()
			self.model_tester.create_and_check_model(*lowerCamelCase_			)

	def    snake_case__(						self:						str			):
			lowercase__					:      Any      =		self.model_tester.prepare_config_and_inputs()
			for type in ["absolute", "relative_key", "relative_key_query"]:
					lowercase__					:      Dict      =		type
					self.model_tester.create_and_check_model(*lowerCamelCase_			)

	def    snake_case__(						self:						Any			):
			lowercase__   ,				lowercase__					:      List[Any]      =		self.model_tester.prepare_config_and_inputs_for_common()
			lowercase__					:      Tuple      =		3
			lowercase__					:      Union[str, Any]      =		input_dict['input_ids']
			lowercase__					:      Union[str, Any]      =		input_ids.ne(1			).to(lowerCamelCase_			)
			lowercase__					:      int      =		ids_tensor([self.model_tester.batch_size],					self.model_tester.type_sequence_label_size			)
			lowercase__					:      Optional[Any]      =		OpenLlamaForSequenceClassification(lowerCamelCase_			)
			model.to(lowerCamelCase_			)
			model.eval()
			lowercase__					:      Optional[Any]      =		model(lowerCamelCase_,					attention_mask=lowerCamelCase_,					labels=lowerCamelCase_			)
			self.assertEqual(result.logits.shape,					(self.model_tester.batch_size, self.model_tester.num_labels)			)

	def    snake_case__(						self:						Any			):
			lowercase__   ,				lowercase__					:      int      =		self.model_tester.prepare_config_and_inputs_for_common()
			lowercase__					:      str      =		3
			lowercase__					:      Optional[Any]      =		'single_label_classification'
			lowercase__					:      Union[str, Any]      =		input_dict['input_ids']
			lowercase__					:      str      =		input_ids.ne(1			).to(lowerCamelCase_			)
			lowercase__					:      Any      =		ids_tensor([self.model_tester.batch_size],					self.model_tester.type_sequence_label_size			)
			lowercase__					:      Dict      =		OpenLlamaForSequenceClassification(lowerCamelCase_			)
			model.to(lowerCamelCase_			)
			model.eval()
			lowercase__					:      Optional[int]      =		model(lowerCamelCase_,					attention_mask=lowerCamelCase_,					labels=lowerCamelCase_			)
			self.assertEqual(result.logits.shape,					(self.model_tester.batch_size, self.model_tester.num_labels)			)

	def    snake_case__(						self:						Union[str, Any]			):
			lowercase__   ,				lowercase__					:      Optional[int]      =		self.model_tester.prepare_config_and_inputs_for_common()
			lowercase__					:      Dict      =		3
			lowercase__					:      List[Any]      =		'multi_label_classification'
			lowercase__					:      Any      =		input_dict['input_ids']
			lowercase__					:      int      =		input_ids.ne(1			).to(lowerCamelCase_			)
			lowercase__					:      Optional[int]      =		ids_tensor(
			    [self.model_tester.batch_size, config.num_labels],					self.model_tester.type_sequence_label_size			).to(torch.float			)
			lowercase__					:      Optional[int]      =		OpenLlamaForSequenceClassification(lowerCamelCase_			)
			model.to(lowerCamelCase_			)
			model.eval()
			lowercase__					:      List[Any]      =		model(lowerCamelCase_,					attention_mask=lowerCamelCase_,					labels=lowerCamelCase_			)
			self.assertEqual(result.logits.shape,					(self.model_tester.batch_size, self.model_tester.num_labels)			)

	@unittest.skip('Open-Llama buffers include complex numbers, which breaks this test'			)
	def    snake_case__(						self:						List[str]			):
			pass





	@parameterized.expand([('linear',), ('dynamic',)]			)
	def    snake_case__(						self:						Any,					lowerCamelCase_:						List[Any]			):
			lowercase__   ,				lowercase__					:      Optional[Any]      =		self.model_tester.prepare_config_and_inputs_for_common()
			lowercase__					:      Optional[int]      =		ids_tensor([1, 10],					config.vocab_size			)
			lowercase__					:      Optional[Any]      =		ids_tensor([1, int(config.max_position_embeddings * 1.5			)],					config.vocab_size			)

			set_seed(42			)  # Fixed seed at init time so the two models get the same random weights
			lowercase__					:      str      =		OpenLlamaModel(lowerCamelCase_			)
			original_model.to(lowerCamelCase_			)
			original_model.eval()
			lowercase__					:      Union[str, Any]      =		original_model(lowerCamelCase_			).last_hidden_state
			lowercase__					:      List[str]      =		original_model(lowerCamelCase_			).last_hidden_state

			set_seed(42			)  # Fixed seed at init time so the two models get the same random weights
			lowercase__					:      List[str]      =		{'type': scaling_type, 'factor': 1_0.0}
			lowercase__					:      Union[str, Any]      =		OpenLlamaModel(lowerCamelCase_			)
			scaled_model.to(lowerCamelCase_			)
			scaled_model.eval()
			lowercase__					:      Dict      =		scaled_model(lowerCamelCase_			).last_hidden_state
			lowercase__					:      Union[str, Any]      =		scaled_model(lowerCamelCase_			).last_hidden_state

			# Dynamic scaling does not change the RoPE embeddings until it receives an input longer than the original
			# maximum sequence length, so the outputs for the short input should match.
			if scaling_type == "dynamic":
					self.assertTrue(torch.allclose(lowerCamelCase_,					lowerCamelCase_,					atol=1E-5			)			)
			else:
					self.assertFalse(torch.allclose(lowerCamelCase_,					lowerCamelCase_,					atol=1E-5			)			)

			# The output should be different for long inputs
			self.assertFalse(torch.allclose(lowerCamelCase_,					lowerCamelCase_,					atol=1E-5			)			)


