


"""simple docstring"""






import unittest

from transformers import LiltConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
	import torch

	from transformers import (
	    LiltForQuestionAnswering,
	    LiltForSequenceClassification,
	    LiltForTokenClassification,
	    LiltModel,
	)
	from transformers.models.lilt.modeling_lilt import LILT_PRETRAINED_MODEL_ARCHIVE_LIST


class 			lowerCamelCase_		:





				"""simple docstring"""







				def __init__(						self       ,		UpperCAmelCase__       ,		UpperCAmelCase__=13       ,		UpperCAmelCase__=7       ,		UpperCAmelCase__=True       ,		UpperCAmelCase__=True       ,		UpperCAmelCase__=True       ,		UpperCAmelCase__=True       ,		UpperCAmelCase__=99       ,		UpperCAmelCase__=24       ,		UpperCAmelCase__=2       ,		UpperCAmelCase__=6       ,		UpperCAmelCase__=37       ,		UpperCAmelCase__="gelu"       ,		UpperCAmelCase__=0.1       ,		UpperCAmelCase__=0.1       ,		UpperCAmelCase__=512       ,		UpperCAmelCase__=16       ,		UpperCAmelCase__=2       ,		UpperCAmelCase__=0.02       ,		UpperCAmelCase__=3       ,		UpperCAmelCase__=None       ,		UpperCAmelCase__=1000       ,		):
								SCREAMING_SNAKE_CASE__     							=				parent
								SCREAMING_SNAKE_CASE__     							=				batch_size
								SCREAMING_SNAKE_CASE__     							=				seq_length
								SCREAMING_SNAKE_CASE__     							=				is_training
								SCREAMING_SNAKE_CASE__     							=				use_input_mask
								SCREAMING_SNAKE_CASE__     							=				use_token_type_ids
								SCREAMING_SNAKE_CASE__     							=				use_labels
								SCREAMING_SNAKE_CASE__     							=				vocab_size
								SCREAMING_SNAKE_CASE__     							=				hidden_size
								SCREAMING_SNAKE_CASE__     							=				num_hidden_layers
								SCREAMING_SNAKE_CASE__     							=				num_attention_heads
								SCREAMING_SNAKE_CASE__     							=				intermediate_size
								SCREAMING_SNAKE_CASE__     							=				hidden_act
								SCREAMING_SNAKE_CASE__     							=				hidden_dropout_prob
								SCREAMING_SNAKE_CASE__     							=				attention_probs_dropout_prob
								SCREAMING_SNAKE_CASE__     							=				max_position_embeddings
								SCREAMING_SNAKE_CASE__     							=				type_vocab_size
								SCREAMING_SNAKE_CASE__     							=				type_sequence_label_size
								SCREAMING_SNAKE_CASE__     							=				initializer_range
								SCREAMING_SNAKE_CASE__     							=				num_labels
								SCREAMING_SNAKE_CASE__     							=				scope
								SCREAMING_SNAKE_CASE__     							=				range_bbox



				def   lowerCAmelCase__		(						self     ):
								SCREAMING_SNAKE_CASE__     							=				ids_tensor([self.batch_size, self.seq_length]       ,		self.vocab_size     )

								SCREAMING_SNAKE_CASE__     							=				ids_tensor([self.batch_size, self.seq_length, 4]       ,		self.range_bbox     )
								# Ensure that bbox is legal
								for i in range(bbox.shape[0]     ):
												for j in range(bbox.shape[1]     ):
																if bbox[i, j, 3] < bbox[i, j, 1]:
																				SCREAMING_SNAKE_CASE__     							=				bbox[i, j, 3]
																				SCREAMING_SNAKE_CASE__     							=				bbox[i, j, 1]
																				SCREAMING_SNAKE_CASE__     							=				t
																if bbox[i, j, 2] < bbox[i, j, 0]:
																				SCREAMING_SNAKE_CASE__     							=				bbox[i, j, 2]
																				SCREAMING_SNAKE_CASE__     							=				bbox[i, j, 0]
																				SCREAMING_SNAKE_CASE__     							=				t

								SCREAMING_SNAKE_CASE__     							=				None
								if self.use_input_mask:
												SCREAMING_SNAKE_CASE__     							=				ids_tensor([self.batch_size, self.seq_length]       ,		vocab_size=2     )

								SCREAMING_SNAKE_CASE__     							=				None
								if self.use_token_type_ids:
												SCREAMING_SNAKE_CASE__     							=				ids_tensor([self.batch_size, self.seq_length]       ,		self.type_vocab_size     )

								SCREAMING_SNAKE_CASE__     							=				None
								SCREAMING_SNAKE_CASE__     							=				None
								if self.use_labels:
												SCREAMING_SNAKE_CASE__     							=				ids_tensor([self.batch_size]       ,		self.type_sequence_label_size     )
												SCREAMING_SNAKE_CASE__     							=				ids_tensor([self.batch_size, self.seq_length]       ,		self.num_labels     )

								SCREAMING_SNAKE_CASE__     							=				self.get_config()

								return config, input_ids, bbox, token_type_ids, input_mask, sequence_labels, token_labels



				def   lowerCAmelCase__		(						self     ):
								return LiltConfig(
								    vocab_size=self.vocab_size       ,		hidden_size=self.hidden_size       ,		num_hidden_layers=self.num_hidden_layers       ,		num_attention_heads=self.num_attention_heads       ,		intermediate_size=self.intermediate_size       ,		hidden_act=self.hidden_act       ,		hidden_dropout_prob=self.hidden_dropout_prob       ,		attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,		max_position_embeddings=self.max_position_embeddings       ,		type_vocab_size=self.type_vocab_size       ,		initializer_range=self.initializer_range       ,		)



				def   lowerCAmelCase__		(						self       ,		UpperCAmelCase__       ,		UpperCAmelCase__       ,		UpperCAmelCase__       ,		UpperCAmelCase__       ,		UpperCAmelCase__       ,		UpperCAmelCase__       ,		UpperCAmelCase__       ,		):
								SCREAMING_SNAKE_CASE__     							=				LiltModel(config=UpperCAmelCase__     )
								model.to(UpperCAmelCase__     )
								model.eval()
								SCREAMING_SNAKE_CASE__     							=				model(UpperCAmelCase__       ,		bbox=UpperCAmelCase__       ,		attention_mask=UpperCAmelCase__       ,		token_type_ids=UpperCAmelCase__     )
								SCREAMING_SNAKE_CASE__     							=				model(UpperCAmelCase__       ,		bbox=UpperCAmelCase__       ,		token_type_ids=UpperCAmelCase__     )
								SCREAMING_SNAKE_CASE__     							=				model(UpperCAmelCase__       ,		bbox=UpperCAmelCase__     )

								self.parent.assertEqual(result.last_hidden_state.shape       ,		(self.batch_size, self.seq_length, self.hidden_size)     )
								self.parent.assertEqual(result.pooler_output.shape       ,		(self.batch_size, self.hidden_size)     )



				def   lowerCAmelCase__		(						self       ,		UpperCAmelCase__       ,		UpperCAmelCase__       ,		UpperCAmelCase__       ,		UpperCAmelCase__       ,		UpperCAmelCase__       ,		UpperCAmelCase__       ,		UpperCAmelCase__       ,		):
								SCREAMING_SNAKE_CASE__     							=				self.num_labels
								SCREAMING_SNAKE_CASE__     							=				LiltForTokenClassification(config=UpperCAmelCase__     )
								model.to(UpperCAmelCase__     )
								model.eval()
								SCREAMING_SNAKE_CASE__     							=				model(
								    UpperCAmelCase__       ,		bbox=UpperCAmelCase__       ,		attention_mask=UpperCAmelCase__       ,		token_type_ids=UpperCAmelCase__       ,		labels=UpperCAmelCase__     )
								self.parent.assertEqual(result.logits.shape       ,		(self.batch_size, self.seq_length, self.num_labels)     )



				def   lowerCAmelCase__		(						self       ,		UpperCAmelCase__       ,		UpperCAmelCase__       ,		UpperCAmelCase__       ,		UpperCAmelCase__       ,		UpperCAmelCase__       ,		UpperCAmelCase__       ,		UpperCAmelCase__       ,		):
								SCREAMING_SNAKE_CASE__     							=				LiltForQuestionAnswering(config=UpperCAmelCase__     )
								model.to(UpperCAmelCase__     )
								model.eval()
								SCREAMING_SNAKE_CASE__     							=				model(
								    UpperCAmelCase__       ,		bbox=UpperCAmelCase__       ,		attention_mask=UpperCAmelCase__       ,		token_type_ids=UpperCAmelCase__       ,		start_positions=UpperCAmelCase__       ,		end_positions=UpperCAmelCase__       ,		)
								self.parent.assertEqual(result.start_logits.shape       ,		(self.batch_size, self.seq_length)     )
								self.parent.assertEqual(result.end_logits.shape       ,		(self.batch_size, self.seq_length)     )






				def   lowerCAmelCase__		(						self     ):
								SCREAMING_SNAKE_CASE__     							=				self.prepare_config_and_inputs()
								(
								    (
								    SCREAMING_SNAKE_CASE__
								)							,      (
								    SCREAMING_SNAKE_CASE__
								)							,      (
								    SCREAMING_SNAKE_CASE__
								)							,      (
								    SCREAMING_SNAKE_CASE__
								)							,      (
								    SCREAMING_SNAKE_CASE__
								)							,      (
								    SCREAMING_SNAKE_CASE__
								)							,      (
								    SCREAMING_SNAKE_CASE__
								)							,      
								)     							=				config_and_inputs
								SCREAMING_SNAKE_CASE__     							=				{
								    "input_ids": input_ids,
								    "bbox": bbox,
								    "token_type_ids": token_type_ids,
								    "attention_mask": input_mask,
								}
								return config, inputs_dict


@require_torch
class 			lowerCamelCase_		(			lowercase       ,  lowercase       ,  lowercase       ,  unittest.TestCase   ):





				"""simple docstring"""







				_lowerCAmelCase      :      Optional[int]      							=					(
				    (
				        LiltModel,
				        LiltForSequenceClassification,
				        LiltForTokenClassification,
				        LiltForQuestionAnswering,
				    )
				    if is_torch_available()
				    else ()
				)
				_lowerCAmelCase      :      Optional[int]      							=					(
				    {
				        "feature-extraction": LiltModel,
				        "question-answering": LiltForQuestionAnswering,
				        "text-classification": LiltForSequenceClassification,
				        "token-classification": LiltForTokenClassification,
				        "zero-shot": LiltForSequenceClassification,
				    }
				    if is_torch_available()
				    else {}
				)
				_lowerCAmelCase      :      List[Any]      							=					False
				_lowerCAmelCase      :      Optional[Any]      							=					False



				def   lowerCAmelCase__		(						self       ,		UpperCAmelCase__       ,		UpperCAmelCase__       ,		UpperCAmelCase__       ,		UpperCAmelCase__       ,		UpperCAmelCase__     ):
								return True



				def   lowerCAmelCase__		(						self     ):
								SCREAMING_SNAKE_CASE__     							=				LiltModelTester(self     )
								SCREAMING_SNAKE_CASE__     							=				ConfigTester(self       ,		config_class=UpperCAmelCase__       ,		hidden_size=37     )



				def   lowerCAmelCase__		(						self     ):
								self.config_tester.run_common_tests()



				def   lowerCAmelCase__		(						self     ):
								SCREAMING_SNAKE_CASE__     							=				self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_model(*UpperCAmelCase__     )



				def   lowerCAmelCase__		(						self     ):
								SCREAMING_SNAKE_CASE__     							=				self.model_tester.prepare_config_and_inputs()
								for type in ["absolute", "relative_key", "relative_key_query"]:
												SCREAMING_SNAKE_CASE__     							=				type
												self.model_tester.create_and_check_model(*UpperCAmelCase__     )



				def   lowerCAmelCase__		(						self     ):
								SCREAMING_SNAKE_CASE__     							=				self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_token_classification(*UpperCAmelCase__     )



				def   lowerCAmelCase__		(						self     ):
								SCREAMING_SNAKE_CASE__     							=				self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_question_answering(*UpperCAmelCase__     )






				@slow
				def   lowerCAmelCase__		(						self     ):
								for model_name in LILT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
												SCREAMING_SNAKE_CASE__     							=				LiltModel.from_pretrained(UpperCAmelCase__     )
												self.assertIsNotNone(UpperCAmelCase__     )


@require_torch
@slow
class 			lowerCamelCase_		(			unittest.TestCase   ):





				"""simple docstring"""







				def   lowerCAmelCase__		(						self     ):
								SCREAMING_SNAKE_CASE__     							=				LiltModel.from_pretrained("SCUT-DLVCLab/lilt-roberta-en-base"     ).to(UpperCAmelCase__     )

								SCREAMING_SNAKE_CASE__     							=				torch.tensor([[1, 2]]       ,		device=UpperCAmelCase__     )
								SCREAMING_SNAKE_CASE__     							=				torch.tensor([[[1, 2, 3, 4], [5, 6, 7, 8]]]       ,		device=UpperCAmelCase__     )

								# forward pass
								with torch.no_grad():
												SCREAMING_SNAKE_CASE__     							=				model(input_ids=UpperCAmelCase__       ,		bbox=UpperCAmelCase__     )

								SCREAMING_SNAKE_CASE__     							=				torch.Size([1, 2, 768]     )
								SCREAMING_SNAKE_CASE__     							=				torch.tensor(
								    [[-0.0_653, 0.0_950, -0.0_061], [-0.0_545, 0.0_926, -0.0_324]]       ,		device=UpperCAmelCase__       ,		)

								self.assertTrue(outputs.last_hidden_state.shape       ,		UpperCAmelCase__     )
								self.assertTrue(torch.allclose(outputs.last_hidden_state[0, :, :3]       ,		UpperCAmelCase__       ,		atol=1e-3     )     )




"""simple docstring"""






import json
from typing import Iterator, List, Union

from tokenizers import AddedToken, Regex, Tokenizer, decoders, normalizers, pre_tokenizers, trainers
from tokenizers.implementations.base_tokenizer import BaseTokenizer
from tokenizers.models import Unigram
from tokenizers.processors import TemplateProcessing


class 			lowerCamelCase_		(			lowercase   ):





				"""simple docstring"""







				def __init__(						self       ,		UpperCAmelCase__ = "▁"       ,		UpperCAmelCase__ = True       ,		UpperCAmelCase__ = "<unk>"       ,		UpperCAmelCase__ = "</s>"       ,		UpperCAmelCase__ = "<pad>"       ,		):
								SCREAMING_SNAKE_CASE__     							=				{
								    "pad": {"id": 0, "token": pad_token},
								    "eos": {"id": 1, "token": eos_token},
								    "unk": {"id": 2, "token": unk_token},
								}

								SCREAMING_SNAKE_CASE__     							=				[None] * len(self.special_tokens     )
								for token_dict in self.special_tokens.values():
												SCREAMING_SNAKE_CASE__     							=				token_dict["token"]

								SCREAMING_SNAKE_CASE__     							=				Tokenizer(Unigram()     )

								SCREAMING_SNAKE_CASE__     							=				normalizers.Sequence(
								    [
								        normalizers.Nmt(),
								        normalizers.NFKC(),
								        normalizers.Replace(Regex(" {2,}"     )       ,		" "     ),
								        normalizers.Lowercase(),
								    ]     )
								SCREAMING_SNAKE_CASE__     							=				pre_tokenizers.Sequence(
								    [
								        pre_tokenizers.Metaspace(replacement=UpperCAmelCase__       ,		add_prefix_space=UpperCAmelCase__     ),
								        pre_tokenizers.Digits(individual_digits=UpperCAmelCase__     ),
								        pre_tokenizers.Punctuation(),
								    ]     )
								SCREAMING_SNAKE_CASE__     							=				decoders.Metaspace(replacement=UpperCAmelCase__       ,		add_prefix_space=UpperCAmelCase__     )

								SCREAMING_SNAKE_CASE__     							=				TemplateProcessing(
								    single=f'''$A {self.special_tokens["eos"]["token"]}'''       ,		special_tokens=[(self.special_tokens["eos"]["token"], self.special_tokens["eos"]["id"])]       ,		)

								SCREAMING_SNAKE_CASE__     							=				{
								    "model": "SentencePieceUnigram",
								    "replacement": replacement,
								    "add_prefix_space": add_prefix_space,
								}

								super().__init__(UpperCAmelCase__       ,		UpperCAmelCase__     )



				def   lowerCAmelCase__		(						self       ,		UpperCAmelCase__       ,		UpperCAmelCase__ = 8000       ,		UpperCAmelCase__ = True       ,		):
								SCREAMING_SNAKE_CASE__     							=				trainers.UnigramTrainer(
								    vocab_size=UpperCAmelCase__       ,		special_tokens=self.special_tokens_list       ,		show_progress=UpperCAmelCase__       ,		)

								if isinstance(UpperCAmelCase__       ,		UpperCAmelCase__     ):
												SCREAMING_SNAKE_CASE__     							=				[files]
								self._tokenizer.train(UpperCAmelCase__       ,		trainer=UpperCAmelCase__     )

								self.add_unk_id()



				def   lowerCAmelCase__		(						self       ,		UpperCAmelCase__       ,		UpperCAmelCase__ = 8000       ,		UpperCAmelCase__ = True       ,		):
								SCREAMING_SNAKE_CASE__     							=				trainers.UnigramTrainer(
								    vocab_size=UpperCAmelCase__       ,		special_tokens=self.special_tokens_list       ,		show_progress=UpperCAmelCase__       ,		)

								self._tokenizer.train_from_iterator(UpperCAmelCase__       ,		trainer=UpperCAmelCase__     )

								self.add_unk_id()






				def   lowerCAmelCase__		(						self     ):
								SCREAMING_SNAKE_CASE__     							=				json.loads(self._tokenizer.to_str()     )

								SCREAMING_SNAKE_CASE__     							=				self.special_tokens["unk"]["id"]

								SCREAMING_SNAKE_CASE__     							=				Tokenizer.from_str(json.dumps(UpperCAmelCase__     )     )
