



def   lowerCAmelCase__						(							_SCREAMING_SNAKE_CASE       :			int    ):
				"""simple docstring"""







				if not isinstance(_SCREAMING_SNAKE_CASE				,     _SCREAMING_SNAKE_CASE    ) or number < 0:
								raise ValueError("""Input must be a non-negative integer"""    )

				__a       =		0
				while number:
								# This way we arrive at next set bit (next 1) instead of looping
								# through each bit and checking for 1s hence the
								# loop won't run 32 times it will only run the number of `1` times
								number &= number - 1
								count += 1
				return count


if __name__ == "__main__":
						import doctest

						doctest.testmod()





from __future__ import annotations

import unittest

from transformers import RoFormerConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
						import tensorflow as tf

						from transformers import (
						    TFRoFormerForCausalLM,
						    TFRoFormerForMaskedLM,
						    TFRoFormerForMultipleChoice,
						    TFRoFormerForQuestionAnswering,
						    TFRoFormerForSequenceClassification,
						    TFRoFormerForTokenClassification,
						    TFRoFormerModel,
						)
						from transformers.models.roformer.modeling_tf_roformer import (
						    TFRoFormerSelfAttention,
						    TFRoFormerSinusoidalPositionalEmbedding,
						)






class    SCREAMING_SNAKE_CASE			:





							def __init__(   self		:		Tuple      ,				__lowercase		:		int      ,				__lowercase		:		str=13      ,				__lowercase		:		Tuple=7      ,				__lowercase		:		int=True      ,				__lowercase		:		Optional[int]=True      ,				__lowercase		:		List[str]=True      ,				__lowercase		:		List[str]=True      ,				__lowercase		:		Any=99      ,				__lowercase		:		int=32      ,				__lowercase		:		Optional[int]=2      ,				__lowercase		:		List[str]=4      ,				__lowercase		:		int=37      ,				__lowercase		:		Optional[int]="gelu"      ,				__lowercase		:		Any=0.1      ,				__lowercase		:		List[Any]=0.1      ,				__lowercase		:		int=512      ,				__lowercase		:		str=16      ,				__lowercase		:		str=2      ,				__lowercase		:		Optional[Any]=0.02      ,				__lowercase		:		str=3      ,				__lowercase		:		str=4      ,				__lowercase		:		str=None      ,				):

											'''simple docstring'''
											__a       =		parent
											__a       =		13
											__a       =		7
											__a       =		True
											__a       =		True
											__a       =		True
											__a       =		True
											__a       =		99
											__a       =		32
											__a       =		2
											__a       =		4
											__a       =		37
											__a       =		"""gelu"""
											__a       =		0.1
											__a       =		0.1
											__a       =		512
											__a       =		16
											__a       =		2
											__a       =		0.02
											__a       =		3
											__a       =		4
											__a       =		None





							def 						UpperCamelCase_					(   self		:		Dict	):

											'''simple docstring'''
											__a       =		ids_tensor([self.batch_size, self.seq_length]      ,				self.vocab_size	)

											__a       =		None
											if self.use_input_mask:
															__a       =		random_attention_mask([self.batch_size, self.seq_length]	)

											__a       =		None
											if self.use_token_type_ids:
															__a       =		ids_tensor([self.batch_size, self.seq_length]      ,				self.type_vocab_size	)

											__a       =		None
											__a       =		None
											__a       =		None
											if self.use_labels:
															__a       =		ids_tensor([self.batch_size]      ,				self.type_sequence_label_size	)
															__a       =		ids_tensor([self.batch_size, self.seq_length]      ,				self.num_labels	)
															__a       =		ids_tensor([self.batch_size]      ,				self.num_choices	)

											__a       =		RoFormerConfig(
											    vocab_size=self.vocab_size      ,				hidden_size=self.hidden_size      ,				num_hidden_layers=self.num_hidden_layers      ,				num_attention_heads=self.num_attention_heads      ,				intermediate_size=self.intermediate_size      ,				hidden_act=self.hidden_act      ,				hidden_dropout_prob=self.hidden_dropout_prob      ,				attention_probs_dropout_prob=self.attention_probs_dropout_prob      ,				max_position_embeddings=self.max_position_embeddings      ,				type_vocab_size=self.type_vocab_size      ,				initializer_range=self.initializer_range      ,				return_dict=__lowercase      ,				)

											return config, input_ids, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels





							def 						UpperCamelCase_					(   self		:		Tuple      ,				__lowercase		:		Optional[int]      ,				__lowercase		:		Tuple      ,				__lowercase		:		Any      ,				__lowercase		:		Tuple      ,				__lowercase		:		int      ,				__lowercase		:		List[Any]      ,				__lowercase		:		List[Any]	):

											'''simple docstring'''
											__a       =		TFRoFormerModel(config=__lowercase	)
											__a       =		{"""input_ids""": input_ids, """attention_mask""": input_mask, """token_type_ids""": token_type_ids}

											__a       =		[input_ids, input_mask]
											__a       =		model(__lowercase	)

											__a       =		model(__lowercase	)

											self.parent.assertEqual(result.last_hidden_state.shape      ,				(self.batch_size, self.seq_length, self.hidden_size)	)





							def 						UpperCamelCase_					(   self		:		Any      ,				__lowercase		:		Union[str, Any]      ,				__lowercase		:		int      ,				__lowercase		:		Any      ,				__lowercase		:		int      ,				__lowercase		:		Any      ,				__lowercase		:		str      ,				__lowercase		:		Optional[int]	):

											'''simple docstring'''
											__a       =		True
											__a       =		TFRoFormerForCausalLM(config=__lowercase	)
											__a       =		{
											    """input_ids""": input_ids,
											    """attention_mask""": input_mask,
											    """token_type_ids""": token_type_ids,
											}
											__a       =		model(__lowercase	)["""logits"""]
											self.parent.assertListEqual(
											    list(prediction_scores.numpy().shape	)      ,				[self.batch_size, self.seq_length, self.vocab_size]	)





							def 						UpperCamelCase_					(   self		:		Union[str, Any]      ,				__lowercase		:		Tuple      ,				__lowercase		:		Tuple      ,				__lowercase		:		Optional[Any]      ,				__lowercase		:		Any      ,				__lowercase		:		int      ,				__lowercase		:		List[str]      ,				__lowercase		:		str	):

											'''simple docstring'''
											__a       =		TFRoFormerForMaskedLM(config=__lowercase	)
											__a       =		{
											    """input_ids""": input_ids,
											    """attention_mask""": input_mask,
											    """token_type_ids""": token_type_ids,
											}
											__a       =		model(__lowercase	)
											self.parent.assertEqual(result.logits.shape      ,				(self.batch_size, self.seq_length, self.vocab_size)	)





							def 						UpperCamelCase_					(   self		:		Optional[Any]      ,				__lowercase		:		int      ,				__lowercase		:		Union[str, Any]      ,				__lowercase		:		Optional[int]      ,				__lowercase		:		str      ,				__lowercase		:		Dict      ,				__lowercase		:		List[str]      ,				__lowercase		:		Optional[int]	):

											'''simple docstring'''
											__a       =		self.num_labels
											__a       =		TFRoFormerForSequenceClassification(config=__lowercase	)
											__a       =		{
											    """input_ids""": input_ids,
											    """attention_mask""": input_mask,
											    """token_type_ids""": token_type_ids,
											}

											__a       =		model(__lowercase	)
											self.parent.assertEqual(result.logits.shape      ,				(self.batch_size, self.num_labels)	)





							def 						UpperCamelCase_					(   self		:		Any      ,				__lowercase		:		Union[str, Any]      ,				__lowercase		:		Tuple      ,				__lowercase		:		List[Any]      ,				__lowercase		:		int      ,				__lowercase		:		Tuple      ,				__lowercase		:		int      ,				__lowercase		:		Any	):

											'''simple docstring'''
											__a       =		self.num_choices
											__a       =		TFRoFormerForMultipleChoice(config=__lowercase	)
											__a       =		tf.tile(tf.expand_dims(__lowercase      ,				1	)      ,				(1, self.num_choices, 1)	)
											__a       =		tf.tile(tf.expand_dims(__lowercase      ,				1	)      ,				(1, self.num_choices, 1)	)
											__a       =		tf.tile(tf.expand_dims(__lowercase      ,				1	)      ,				(1, self.num_choices, 1)	)
											__a       =		{
											    """input_ids""": multiple_choice_inputs_ids,
											    """attention_mask""": multiple_choice_input_mask,
											    """token_type_ids""": multiple_choice_token_type_ids,
											}
											__a       =		model(__lowercase	)
											self.parent.assertEqual(result.logits.shape      ,				(self.batch_size, self.num_choices)	)





							def 						UpperCamelCase_					(   self		:		List[Any]      ,				__lowercase		:		Optional[Any]      ,				__lowercase		:		Dict      ,				__lowercase		:		List[str]      ,				__lowercase		:		str      ,				__lowercase		:		int      ,				__lowercase		:		Dict      ,				__lowercase		:		List[str]	):

											'''simple docstring'''
											__a       =		self.num_labels
											__a       =		TFRoFormerForTokenClassification(config=__lowercase	)
											__a       =		{
											    """input_ids""": input_ids,
											    """attention_mask""": input_mask,
											    """token_type_ids""": token_type_ids,
											}
											__a       =		model(__lowercase	)
											self.parent.assertEqual(result.logits.shape      ,				(self.batch_size, self.seq_length, self.num_labels)	)





							def 						UpperCamelCase_					(   self		:		Optional[Any]      ,				__lowercase		:		List[Any]      ,				__lowercase		:		Dict      ,				__lowercase		:		Optional[Any]      ,				__lowercase		:		List[str]      ,				__lowercase		:		Any      ,				__lowercase		:		str      ,				__lowercase		:		Dict	):

											'''simple docstring'''
											__a       =		TFRoFormerForQuestionAnswering(config=__lowercase	)
											__a       =		{
											    """input_ids""": input_ids,
											    """attention_mask""": input_mask,
											    """token_type_ids""": token_type_ids,
											}

											__a       =		model(__lowercase	)
											self.parent.assertEqual(result.start_logits.shape      ,				(self.batch_size, self.seq_length)	)
											self.parent.assertEqual(result.end_logits.shape      ,				(self.batch_size, self.seq_length)	)







							def 						UpperCamelCase_					(   self		:		Union[str, Any]	):

											'''simple docstring'''
											__a       =		self.prepare_config_and_inputs()
											(
											    (
											    __a
											)     ,					(
											    __a
											)     ,					(
											    __a
											)     ,					(
											    __a
											)     ,					(
											    __a
											)     ,					(
											    __a
											)     ,					(
											    __a
											)     ,					
											)       =		config_and_inputs
											__a       =		{"""input_ids""": input_ids, """token_type_ids""": token_type_ids, """attention_mask""": input_mask}
											return config, inputs_dict






@require_tf
class    SCREAMING_SNAKE_CASE			(							lowerCamelCase__			,      lowerCamelCase__			,      unittest.TestCase					):
							__lowerCamelCase	:	Optional[int]    =(
							    (
							        TFRoFormerModel,
							        TFRoFormerForCausalLM,
							        TFRoFormerForMaskedLM,
							        TFRoFormerForQuestionAnswering,
							        TFRoFormerForSequenceClassification,
							        TFRoFormerForTokenClassification,
							        TFRoFormerForMultipleChoice,
							    )
							    if is_tf_available()
							    else ()
							)
							__lowerCamelCase	:	Optional[int]    =(
							    {
							        'feature-extraction': TFRoFormerModel,
							        'fill-mask': TFRoFormerForMaskedLM,
							        'question-answering': TFRoFormerForQuestionAnswering,
							        'text-classification': TFRoFormerForSequenceClassification,
							        'text-generation': TFRoFormerForCausalLM,
							        'token-classification': TFRoFormerForTokenClassification,
							        'zero-shot': TFRoFormerForSequenceClassification,
							    }
							    if is_tf_available()
							    else {}
							)

							__lowerCamelCase	:	Optional[int]    =False
							__lowerCamelCase	:	Tuple    =False





							def 						UpperCamelCase_					(   self		:		Any      ,				__lowercase		:		int      ,				__lowercase		:		List[Any]      ,				__lowercase		:		Optional[Any]      ,				__lowercase		:		Optional[int]      ,				__lowercase		:		Tuple	):

											'''simple docstring'''
											if pipeline_test_casse_name == "TextGenerationPipelineTests":
															return True

											return False





							def 						UpperCamelCase_					(   self		:		int	):

											'''simple docstring'''
											__a       =		TFRoFormerModelTester(self	)
											__a       =		ConfigTester(self      ,				config_class=__lowercase      ,				hidden_size=37	)





							def 						UpperCamelCase_					(   self		:		Optional[int]	):

											'''simple docstring'''
											self.config_tester.run_common_tests()





							def 						UpperCamelCase_					(   self		:		List[Any]	):

											'''simple docstring'''
											__a       =		self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_model(*__lowercase	)





							def 						UpperCamelCase_					(   self		:		str	):

											'''simple docstring'''
											__a       =		self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_masked_lm(*__lowercase	)





							def 						UpperCamelCase_					(   self		:		Dict	):

											'''simple docstring'''
											__a       =		self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_lm_head(*__lowercase	)





							def 						UpperCamelCase_					(   self		:		List[Any]	):

											'''simple docstring'''
											__a       =		self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_multiple_choice(*__lowercase	)





							def 						UpperCamelCase_					(   self		:		Optional[Any]	):

											'''simple docstring'''
											__a       =		self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_question_answering(*__lowercase	)





							def 						UpperCamelCase_					(   self		:		List[Any]	):

											'''simple docstring'''
											__a       =		self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_sequence_classification(*__lowercase	)





							def 						UpperCamelCase_					(   self		:		Dict	):

											'''simple docstring'''
											__a       =		self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_token_classification(*__lowercase	)







							@slow
							def 						UpperCamelCase_					(   self		:		List[Any]	):

											'''simple docstring'''
											__a       =		TFRoFormerModel.from_pretrained("""junnyu/roformer_chinese_base"""	)
											self.assertIsNotNone(__lowercase	)






@require_tf
class    SCREAMING_SNAKE_CASE			(							unittest.TestCase					):





							@slow
							def 						UpperCamelCase_					(   self		:		int	):

											'''simple docstring'''
											__a       =		TFRoFormerForMaskedLM.from_pretrained("""junnyu/roformer_chinese_base"""	)
											__a       =		tf.constant([[0, 1, 2, 3, 4, 5]]	)
											__a       =		model(__lowercase	)[0]

											# TODO Replace vocab size
											__a       =		50000

											__a       =		[1, 6, vocab_size]
											self.assertEqual(output.shape      ,				__lowercase	)

											print(output[:, :3, :3]	)

											# TODO Replace values below with what was printed above.
											__a       =		tf.constant(
											    [
											        [
											            [-0.12053341, -1.0264901, 0.29221946],
											            [-1.5133783, 0.197433, 0.15190607],
											            [-5.0135403, -3.900256, -0.84038764],
											        ]
											    ]	)
											tf.debugging.assert_near(output[:, :3, :3]      ,				__lowercase      ,				atol=1E-4	)






@require_tf
class    SCREAMING_SNAKE_CASE			(							unittest.TestCase					):
							__lowerCamelCase	:	Dict    =1e-4





							def 						UpperCamelCase_					(   self		:		str	):

											'''simple docstring'''
											__a       =		tf.constant([[4, 10]]	)
											__a       =		TFRoFormerSinusoidalPositionalEmbedding(num_positions=6      ,				embedding_dim=6	)

											__a       =		emba(input_ids.shape	)
											__a       =		tf.constant(
											    [[0.0000, 0.0000, 0.0000, 1.0000, 1.0000, 1.0000], [0.8415, 0.0464, 0.0022, 0.5403, 0.9989, 1.0000]]	)

											tf.debugging.assert_near(__lowercase      ,				__lowercase      ,				atol=self.tolerance	)







							def 						UpperCamelCase_					(   self		:		str	):

											'''simple docstring'''
											__a       =		tf.constant(
											    [
											        [0.0000, 0.0000, 0.0000, 0.0000, 0.0000],
											        [0.8415, 0.8219, 0.8020, 0.7819, 0.7617],
											        [0.9093, 0.9364, 0.9581, 0.9749, 0.9870],
											    ]	)
											__a       =		TFRoFormerSinusoidalPositionalEmbedding(num_positions=512      ,				embedding_dim=512	)
											emba([2, 16, 512]	)
											__a       =		emba.weight[:3, :5]

											tf.debugging.assert_near(__lowercase      ,				__lowercase      ,				atol=self.tolerance	)






@require_tf
class    SCREAMING_SNAKE_CASE			(							unittest.TestCase					):
							__lowerCamelCase	:	int    =1e-4





							def 						UpperCamelCase_					(   self		:		str	):

											'''simple docstring'''
											# 2,12,16,64
											__a       =		tf.reshape(tf.range(2 * 12 * 16 * 64      ,				dtype=tf.floataa	)      ,				shape=(2, 12, 16, 64)	) / 100

											__a       =		-tf.reshape(tf.range(2 * 12 * 16 * 64      ,				dtype=tf.floataa	)      ,				shape=(2, 12, 16, 64)	) / 100

											__a       =		TFRoFormerSinusoidalPositionalEmbedding(num_positions=32      ,				embedding_dim=64	)
											__a       =		embed_positions([2, 16, 768]	)[None, None, :, :]

											__a     ,					__a       =		TFRoFormerSelfAttention.apply_rotary_position_embeddings(
											    __lowercase      ,				__lowercase      ,				__lowercase	)

											__a       =		tf.constant(
											    [
											        [0.0000, 0.0100, 0.0200, 0.0300, 0.0400, 0.0500, 0.0600, 0.0700],
											        [-0.2012, 0.8897, 0.0263, 0.9401, 0.2074, 0.9463, 0.3481, 0.9343],
											        [-1.7057, 0.6271, -1.2145, 1.3897, -0.6303, 1.7647, -0.1173, 1.8985],
											        [-2.1731, -1.6397, -2.7358, 0.2854, -2.1840, 1.7183, -1.3018, 2.4871],
											        [0.2717, -3.6173, -2.9206, -2.1988, -3.6638, 0.3858, -2.9155, 2.2980],
											        [3.9859, -2.1580, -0.7984, -4.4904, -4.1181, -2.0252, -4.4782, 1.1253],
											    ]	)
											__a       =		tf.constant(
											    [
											        [0.0000, -0.0100, -0.0200, -0.0300, -0.0400, -0.0500, -0.0600, -0.0700],
											        [0.2012, -0.8897, -0.0263, -0.9401, -0.2074, -0.9463, -0.3481, -0.9343],
											        [1.7057, -0.6271, 1.2145, -1.3897, 0.6303, -1.7647, 0.1173, -1.8985],
											        [2.1731, 1.6397, 2.7358, -0.2854, 2.1840, -1.7183, 1.3018, -2.4871],
											        [-0.2717, 3.6173, 2.9206, 2.1988, 3.6638, -0.3858, 2.9155, -2.2980],
											        [-3.9859, 2.1580, 0.7984, 4.4904, 4.1181, 2.0252, 4.4782, -1.1253],
											    ]	)

											tf.debugging.assert_near(query_layer[0, 0, :6, :8]      ,				__lowercase      ,				atol=self.tolerance	)
											tf.debugging.assert_near(key_layer[0, 0, :6, :8]      ,				__lowercase      ,				atol=self.tolerance	)
