




import unittest

from transformers import LiltConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
			import torch

			from transformers import (
			    LiltForQuestionAnswering,
			    LiltForSequenceClassification,
			    LiltForTokenClassification,
			    LiltModel,
			)
			from transformers.models.lilt.modeling_lilt import LILT_PRETRAINED_MODEL_ARCHIVE_LIST




class       _A				:






		def __init__(    self						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE=13						,		_SCREAMING_SNAKE_CASE=7						,		_SCREAMING_SNAKE_CASE=True						,		_SCREAMING_SNAKE_CASE=True						,		_SCREAMING_SNAKE_CASE=True						,		_SCREAMING_SNAKE_CASE=True						,		_SCREAMING_SNAKE_CASE=99						,		_SCREAMING_SNAKE_CASE=24						,		_SCREAMING_SNAKE_CASE=2						,		_SCREAMING_SNAKE_CASE=6						,		_SCREAMING_SNAKE_CASE=37						,		_SCREAMING_SNAKE_CASE="gelu"						,		_SCREAMING_SNAKE_CASE=0.1						,		_SCREAMING_SNAKE_CASE=0.1						,		_SCREAMING_SNAKE_CASE=512						,		_SCREAMING_SNAKE_CASE=16						,		_SCREAMING_SNAKE_CASE=2						,		_SCREAMING_SNAKE_CASE=0.02						,		_SCREAMING_SNAKE_CASE=3						,		_SCREAMING_SNAKE_CASE=None						,		_SCREAMING_SNAKE_CASE=1000						,		):
								_UpperCAmelCase      =			parent
								_UpperCAmelCase      =			batch_size
								_UpperCAmelCase      =			seq_length
								_UpperCAmelCase      =			is_training
								_UpperCAmelCase      =			use_input_mask
								_UpperCAmelCase      =			use_token_type_ids
								_UpperCAmelCase      =			use_labels
								_UpperCAmelCase      =			vocab_size
								_UpperCAmelCase      =			hidden_size
								_UpperCAmelCase      =			num_hidden_layers
								_UpperCAmelCase      =			num_attention_heads
								_UpperCAmelCase      =			intermediate_size
								_UpperCAmelCase      =			hidden_act
								_UpperCAmelCase      =			hidden_dropout_prob
								_UpperCAmelCase      =			attention_probs_dropout_prob
								_UpperCAmelCase      =			max_position_embeddings
								_UpperCAmelCase      =			type_vocab_size
								_UpperCAmelCase      =			type_sequence_label_size
								_UpperCAmelCase      =			initializer_range
								_UpperCAmelCase      =			num_labels
								_UpperCAmelCase      =			scope
								_UpperCAmelCase      =			range_bbox






		def 				UpperCAmelCase (    self			):
								_UpperCAmelCase      =			ids_tensor([self.batch_size, self.seq_length]						,		self.vocab_size			)

								_UpperCAmelCase      =			ids_tensor([self.batch_size, self.seq_length, 4]						,		self.range_bbox			)
								# Ensure that bbox is legal
								for i in range(bbox.shape[0]			):
														for j in range(bbox.shape[1]			):
																				if bbox[i, j, 3] < bbox[i, j, 1]:
																										_UpperCAmelCase      =			bbox[i, j, 3]
																										_UpperCAmelCase      =			bbox[i, j, 1]
																										_UpperCAmelCase      =			t
																				if bbox[i, j, 2] < bbox[i, j, 0]:
																										_UpperCAmelCase      =			bbox[i, j, 2]
																										_UpperCAmelCase      =			bbox[i, j, 0]
																										_UpperCAmelCase      =			t

								_UpperCAmelCase      =			None
								if self.use_input_mask:
														_UpperCAmelCase      =			ids_tensor([self.batch_size, self.seq_length]						,		vocab_size=2			)

								_UpperCAmelCase      =			None
								if self.use_token_type_ids:
														_UpperCAmelCase      =			ids_tensor([self.batch_size, self.seq_length]						,		self.type_vocab_size			)

								_UpperCAmelCase      =			None
								_UpperCAmelCase      =			None
								if self.use_labels:
														_UpperCAmelCase      =			ids_tensor([self.batch_size]						,		self.type_sequence_label_size			)
														_UpperCAmelCase      =			ids_tensor([self.batch_size, self.seq_length]						,		self.num_labels			)

								_UpperCAmelCase      =			self.get_config()

								return config, input_ids, bbox, token_type_ids, input_mask, sequence_labels, token_labels






		def 				UpperCAmelCase (    self			):
								return LiltConfig(
								    vocab_size=self.vocab_size						,		hidden_size=self.hidden_size						,		num_hidden_layers=self.num_hidden_layers						,		num_attention_heads=self.num_attention_heads						,		intermediate_size=self.intermediate_size						,		hidden_act=self.hidden_act						,		hidden_dropout_prob=self.hidden_dropout_prob						,		attention_probs_dropout_prob=self.attention_probs_dropout_prob						,		max_position_embeddings=self.max_position_embeddings						,		type_vocab_size=self.type_vocab_size						,		initializer_range=self.initializer_range						,		)






		def 				UpperCAmelCase (    self						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE						,		):
								_UpperCAmelCase      =			LiltModel(config=UpperCAmelCase_			)
								model.to(UpperCAmelCase_			)
								model.eval()
								_UpperCAmelCase      =			model(UpperCAmelCase_						,		bbox=UpperCAmelCase_						,		attention_mask=UpperCAmelCase_						,		token_type_ids=UpperCAmelCase_			)
								_UpperCAmelCase      =			model(UpperCAmelCase_						,		bbox=UpperCAmelCase_						,		token_type_ids=UpperCAmelCase_			)
								_UpperCAmelCase      =			model(UpperCAmelCase_						,		bbox=UpperCAmelCase_			)

								self.parent.assertEqual(result.last_hidden_state.shape						,		(self.batch_size, self.seq_length, self.hidden_size)			)
								self.parent.assertEqual(result.pooler_output.shape						,		(self.batch_size, self.hidden_size)			)






		def 				UpperCAmelCase (    self						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE						,		):
								_UpperCAmelCase      =			self.num_labels
								_UpperCAmelCase      =			LiltForTokenClassification(config=UpperCAmelCase_			)
								model.to(UpperCAmelCase_			)
								model.eval()
								_UpperCAmelCase      =			model(
								    UpperCAmelCase_						,		bbox=UpperCAmelCase_						,		attention_mask=UpperCAmelCase_						,		token_type_ids=UpperCAmelCase_						,		labels=UpperCAmelCase_			)
								self.parent.assertEqual(result.logits.shape						,		(self.batch_size, self.seq_length, self.num_labels)			)






		def 				UpperCAmelCase (    self						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE						,		):
								_UpperCAmelCase      =			LiltForQuestionAnswering(config=UpperCAmelCase_			)
								model.to(UpperCAmelCase_			)
								model.eval()
								_UpperCAmelCase      =			model(
								    UpperCAmelCase_						,		bbox=UpperCAmelCase_						,		attention_mask=UpperCAmelCase_						,		token_type_ids=UpperCAmelCase_						,		start_positions=UpperCAmelCase_						,		end_positions=UpperCAmelCase_						,		)
								self.parent.assertEqual(result.start_logits.shape						,		(self.batch_size, self.seq_length)			)
								self.parent.assertEqual(result.end_logits.shape						,		(self.batch_size, self.seq_length)			)






		def 				UpperCAmelCase (    self			):
								_UpperCAmelCase      =			self.prepare_config_and_inputs()
								(
								    (
								    _UpperCAmelCase						
								)    ,					(
								    _UpperCAmelCase						
								)    ,					(
								    _UpperCAmelCase						
								)    ,					(
								    _UpperCAmelCase						
								)    ,					(
								    _UpperCAmelCase						
								)    ,					(
								    _UpperCAmelCase						
								)    ,					(
								    _UpperCAmelCase						
								)    ,											
								)      =			config_and_inputs
								_UpperCAmelCase      =			{
								    """input_ids""": input_ids,
								    """bbox""": bbox,
								    """token_type_ids""": token_type_ids,
								    """attention_mask""": input_mask,
								}
								return config, inputs_dict




@require_torch
class       _A				(		snake_case__       ,     snake_case__       ,     snake_case__       ,     unittest.TestCase     ):
		__a					    =       (
		    (
		        LiltModel,
		        LiltForSequenceClassification,
		        LiltForTokenClassification,
		        LiltForQuestionAnswering,
		    )
		    if is_torch_available()
		    else ()
		)
		__a					    =       (
		    {
		        """feature-extraction""": LiltModel,
		        """question-answering""": LiltForQuestionAnswering,
		        """text-classification""": LiltForSequenceClassification,
		        """token-classification""": LiltForTokenClassification,
		        """zero-shot""": LiltForSequenceClassification,
		    }
		    if is_torch_available()
		    else {}
		)
		__a					    =       False
		__a					    =       False






		def 				UpperCAmelCase (    self						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE						,		_SCREAMING_SNAKE_CASE			):
								return True






		def 				UpperCAmelCase (    self			):
								_UpperCAmelCase      =			LiltModelTester(self			)
								_UpperCAmelCase      =			ConfigTester(self						,		config_class=UpperCAmelCase_						,		hidden_size=37			)






		def 				UpperCAmelCase (    self			):
								self.config_tester.run_common_tests()






		def 				UpperCAmelCase (    self			):
								_UpperCAmelCase      =			self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_model(*UpperCAmelCase_			)






		def 				UpperCAmelCase (    self			):
								_UpperCAmelCase      =			self.model_tester.prepare_config_and_inputs()
								for type in ["absolute", "relative_key", "relative_key_query"]:
														_UpperCAmelCase      =			type
														self.model_tester.create_and_check_model(*UpperCAmelCase_			)






		def 				UpperCAmelCase (    self			):
								_UpperCAmelCase      =			self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_token_classification(*UpperCAmelCase_			)






		def 				UpperCAmelCase (    self			):
								_UpperCAmelCase      =			self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_question_answering(*UpperCAmelCase_			)






		@slow
		def 				UpperCAmelCase (    self			):



								for model_name in LILT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
														_UpperCAmelCase      =			LiltModel.from_pretrained(UpperCAmelCase_			)
														self.assertIsNotNone(UpperCAmelCase_			)




@require_torch
@slow
class       _A				(		unittest.TestCase     ):






		def 				UpperCAmelCase (    self			):
								_UpperCAmelCase      =			LiltModel.from_pretrained("""SCUT-DLVCLab/lilt-roberta-en-base"""			).to(UpperCAmelCase_			)

								_UpperCAmelCase      =			torch.tensor([[1, 2]]						,		device=UpperCAmelCase_			)
								_UpperCAmelCase      =			torch.tensor([[[1, 2, 3, 4], [5, 6, 7, 8]]]						,		device=UpperCAmelCase_			)

								# forward pass
								with torch.no_grad():
														_UpperCAmelCase      =			model(input_ids=UpperCAmelCase_						,		bbox=UpperCAmelCase_			)

								_UpperCAmelCase      =			torch.Size([1, 2, 768]			)
								_UpperCAmelCase      =			torch.tensor(
								    [[-0.0653, 0.0950, -0.0061], [-0.0545, 0.0926, -0.0324]]						,		device=UpperCAmelCase_						,		)

								self.assertTrue(outputs.last_hidden_state.shape						,		UpperCAmelCase_			)
								self.assertTrue(torch.allclose(outputs.last_hidden_state[0, :, :3]						,		UpperCAmelCase_						,		atol=1e-3			)			)





import logging

from transformers.configuration_utils import PretrainedConfig


a													=			logging.getLogger(__name__)




class       _A				(		__lowercase     ):
		__a					    =       """masked_bert"""






		def __init__(    self						,		_SCREAMING_SNAKE_CASE=3_0522						,		_SCREAMING_SNAKE_CASE=768						,		_SCREAMING_SNAKE_CASE=12						,		_SCREAMING_SNAKE_CASE=12						,		_SCREAMING_SNAKE_CASE=3072						,		_SCREAMING_SNAKE_CASE="gelu"						,		_SCREAMING_SNAKE_CASE=0.1						,		_SCREAMING_SNAKE_CASE=0.1						,		_SCREAMING_SNAKE_CASE=512						,		_SCREAMING_SNAKE_CASE=2						,		_SCREAMING_SNAKE_CASE=0.02						,		_SCREAMING_SNAKE_CASE=1e-12						,		_SCREAMING_SNAKE_CASE=0						,		_SCREAMING_SNAKE_CASE="topK"						,		_SCREAMING_SNAKE_CASE="constant"						,		_SCREAMING_SNAKE_CASE=0.0						,		**_SCREAMING_SNAKE_CASE						,		):
								super().__init__(pad_token_id=_SCREAMING_SNAKE_CASE						,		**_SCREAMING_SNAKE_CASE			)

								_UpperCAmelCase      =			vocab_size
								_UpperCAmelCase      =			hidden_size
								_UpperCAmelCase      =			num_hidden_layers
								_UpperCAmelCase      =			num_attention_heads
								_UpperCAmelCase      =			hidden_act
								_UpperCAmelCase      =			intermediate_size
								_UpperCAmelCase      =			hidden_dropout_prob
								_UpperCAmelCase      =			attention_probs_dropout_prob
								_UpperCAmelCase      =			max_position_embeddings
								_UpperCAmelCase      =			type_vocab_size
								_UpperCAmelCase      =			initializer_range
								_UpperCAmelCase      =			layer_norm_eps
								_UpperCAmelCase      =			pruning_method
								_UpperCAmelCase      =			mask_init
								_UpperCAmelCase      =			mask_scale