


import os
import unicodedata
from shutil import copyfile
from typing import Any, Dict, List, Optional, Tuple

import sentencepiece as spm

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


lowerCAmelCase       		=    logging.get_logger(__name__)
lowerCAmelCase       		=    {"""vocab_file""": """spiece.model"""}

lowerCAmelCase       		=    {
    """vocab_file""": {
        """albert-base-v1""": """https://huggingface.co/albert-base-v1/resolve/main/spiece.model""",
        """albert-large-v1""": """https://huggingface.co/albert-large-v1/resolve/main/spiece.model""",
        """albert-xlarge-v1""": """https://huggingface.co/albert-xlarge-v1/resolve/main/spiece.model""",
        """albert-xxlarge-v1""": """https://huggingface.co/albert-xxlarge-v1/resolve/main/spiece.model""",
        """albert-base-v2""": """https://huggingface.co/albert-base-v2/resolve/main/spiece.model""",
        """albert-large-v2""": """https://huggingface.co/albert-large-v2/resolve/main/spiece.model""",
        """albert-xlarge-v2""": """https://huggingface.co/albert-xlarge-v2/resolve/main/spiece.model""",
        """albert-xxlarge-v2""": """https://huggingface.co/albert-xxlarge-v2/resolve/main/spiece.model""",
    }
}

lowerCAmelCase       		=    {
    """albert-base-v1""": 512,
    """albert-large-v1""": 512,
    """albert-xlarge-v1""": 512,
    """albert-xxlarge-v1""": 512,
    """albert-base-v2""": 512,
    """albert-large-v2""": 512,
    """albert-xlarge-v2""": 512,
    """albert-xxlarge-v2""": 512,
}

lowerCAmelCase       		=    """▁"""




class 							lowerCamelCase   (  _UpperCamelCase ):
	_lowerCAmelCase     :		str										=						VOCAB_FILES_NAMES
	_lowerCAmelCase     :		Any										=						PRETRAINED_VOCAB_FILES_MAP
	_lowerCAmelCase     :		Dict										=						PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES


	def __init__(    self				,     lowercase__				,     lowercase__=True				,     lowercase__=True				,     lowercase__=False				,     lowercase__="[CLS]"				,     lowercase__="[SEP]"				,     lowercase__="<unk>"				,     lowercase__="[SEP]"				,     lowercase__="<pad>"				,     lowercase__="[CLS]"				,     lowercase__="[MASK]"				,     lowercase__ = None				,     **lowercase__				,     ):
							# Mask token behave like a normal word, i.e. include the space before it and
							# is included in the raw text, there should be a match in a non-normalized sentence.
							__UpperCAmelCase			:							Any	    =	(
							    AddedToken(lowercase__				,     lstrip=lowercase__				,     rstrip=lowercase__				,     normalized=lowercase__)
							    if isinstance(lowercase__				,     lowercase__)
							    else mask_token
							)

							__UpperCAmelCase			:							Optional[int]	    =	{} if sp_model_kwargs is None else sp_model_kwargs

							super().__init__(
							    do_lower_case=lowercase__				,     remove_space=lowercase__				,     keep_accents=lowercase__				,     bos_token=lowercase__				,     eos_token=lowercase__				,     unk_token=lowercase__				,     sep_token=lowercase__				,     pad_token=lowercase__				,     cls_token=lowercase__				,     mask_token=lowercase__				,     sp_model_kwargs=self.sp_model_kwargs				,     **lowercase__				,     )

							__UpperCAmelCase			:							Optional[Any]	    =	do_lower_case
							__UpperCAmelCase			:							List[str]	    =	remove_space
							__UpperCAmelCase			:							Dict	    =	keep_accents
							__UpperCAmelCase			:							str	    =	vocab_file

							__UpperCAmelCase			:							Optional[int]	    =	spm.SentencePieceProcessor(**self.sp_model_kwargs)
							self.sp_model.Load(lowercase__)


	@property
	def  A(    self):
							return len(self.sp_model)


	def  A(    self):
							__UpperCAmelCase			:							Union[str, Any]	    =	{self.convert_ids_to_tokens(lowercase__): i for i in range(self.vocab_size)}
							vocab.update(self.added_tokens_encoder)
							return vocab


	def __getstate__(    self):
							__UpperCAmelCase			:							List[Any]	    =	self.__dict__.copy()
							__UpperCAmelCase			:							str	    =	None
							return state


	def __setstate__(    self				,     lowercase__):
							__UpperCAmelCase			:							Optional[Any]	    =	d

							# for backward compatibility
							if not hasattr(self				,     '''sp_model_kwargs'''):
													__UpperCAmelCase			:							List[Any]	    =	{}

							__UpperCAmelCase			:							Dict	    =	spm.SentencePieceProcessor(**self.sp_model_kwargs)
							self.sp_model.Load(self.vocab_file)


	def  A(    self				,     lowercase__):
							if self.remove_space:
													__UpperCAmelCase			:							Optional[Any]	    =	''' '''.join(inputs.strip().split())
							else:
													__UpperCAmelCase			:							str	    =	inputs
							__UpperCAmelCase			:							Tuple	    =	outputs.replace('''``'''				,     '''"''').replace('''\'\''''				,     '''"''')

							if not self.keep_accents:
													__UpperCAmelCase			:							Dict	    =	unicodedata.normalize('''NFKD'''				,     lowercase__)
													__UpperCAmelCase			:							int	    =	''''''.join([c for c in outputs if not unicodedata.combining(lowercase__)])
							if self.do_lower_case:
													__UpperCAmelCase			:							Dict	    =	outputs.lower()

							return outputs


	def  A(    self				,     lowercase__):
							__UpperCAmelCase			:							Tuple	    =	self.preprocess_text(lowercase__)
							__UpperCAmelCase			:							Tuple	    =	self.sp_model.encode(lowercase__				,     out_type=lowercase__)
							__UpperCAmelCase			:							Any	    =	[]
							for piece in pieces:
													if len(lowercase__) > 1 and piece[-1] == str(''',''') and piece[-2].isdigit():
																			__UpperCAmelCase			:							List[Any]	    =	self.sp_model.EncodeAsPieces(piece[:-1].replace(lowercase__				,     ''''''))
																			if piece[0] != SPIECE_UNDERLINE and cur_pieces[0][0] == SPIECE_UNDERLINE:
																									if len(cur_pieces[0]) == 1:
																															__UpperCAmelCase			:							Optional[int]	    =	cur_pieces[1:]
																									else:
																															__UpperCAmelCase			:							List[str]	    =	cur_pieces[0][1:]
																			cur_pieces.append(piece[-1])
																			new_pieces.extend(lowercase__)
													else:
																			new_pieces.append(lowercase__)

							return new_pieces


	def  A(    self				,     lowercase__):
							return self.sp_model.PieceToId(lowercase__)


	def  A(    self				,     lowercase__):
							return self.sp_model.IdToPiece(lowercase__)


	def  A(    self				,     lowercase__):
							__UpperCAmelCase			:							Optional[Any]	    =	[]
							__UpperCAmelCase			:							List[Any]	    =	''''''
							__UpperCAmelCase			:							Tuple	    =	False
							for token in tokens:
													# make sure that special tokens are not decoded using sentencepiece model
													if token in self.all_special_tokens:
																			if not prev_is_special:
																									out_string += " "
																			out_string += self.sp_model.decode(lowercase__) + token
																			__UpperCAmelCase			:							List[Any]	    =	True
																			__UpperCAmelCase			:							str	    =	[]
													else:
																			current_sub_tokens.append(lowercase__)
																			__UpperCAmelCase			:							Dict	    =	False
							out_string += self.sp_model.decode(lowercase__)
							return out_string.strip()


	def  A(    self				,     lowercase__				,     lowercase__ = None):
							__UpperCAmelCase			:							Optional[int]	    =	[self.sep_token_id]
							__UpperCAmelCase			:							Union[str, Any]	    =	[self.cls_token_id]
							if token_ids_a is None:
													return cls + token_ids_a + sep
							return cls + token_ids_a + sep + token_ids_a + sep


	def  A(    self				,     lowercase__				,     lowercase__ = None				,     lowercase__ = False):

							if already_has_special_tokens:
													return super().get_special_tokens_mask(
													    token_ids_a=lowercase__				,     token_ids_a=lowercase__				,     already_has_special_tokens=lowercase__)

							if token_ids_a is not None:
													return [1] + ([0] * len(lowercase__)) + [1] + ([0] * len(lowercase__)) + [1]
							return [1] + ([0] * len(lowercase__)) + [1]


	def  A(    self				,     lowercase__				,     lowercase__ = None):
							__UpperCAmelCase			:							Any	    =	[self.sep_token_id]
							__UpperCAmelCase			:							Any	    =	[self.cls_token_id]

							if token_ids_a is None:
													return len(cls + token_ids_a + sep) * [0]
							return len(cls + token_ids_a + sep) * [0] + len(token_ids_a + sep) * [1]


	def  A(    self				,     lowercase__				,     lowercase__ = None):
							if not os.path.isdir(lowercase__):
													logger.error(F"Vocabulary path ({save_directory}) should be a directory")
													return
							__UpperCAmelCase			:							Any	    =	os.path.join(
							    lowercase__				,     (filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file'''])

							if os.path.abspath(self.vocab_file) != os.path.abspath(lowercase__) and os.path.isfile(self.vocab_file):
													copyfile(self.vocab_file				,     lowercase__)
							elif not os.path.isfile(self.vocab_file):
													with open(lowercase__				,     '''wb''') as fi:
																			__UpperCAmelCase			:							Dict	    =	self.sp_model.serialized_model_proto()
																			fi.write(lowercase__)

							return (out_vocab_file,)





import unittest

from transformers import LiltConfig, is_torch_available
from transformers.testing_utils import require_torch, slow, torch_device

from ...generation.test_utils import GenerationTesterMixin
from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
				import torch

				from transformers import (
				    LiltForQuestionAnswering,
				    LiltForSequenceClassification,
				    LiltForTokenClassification,
				    LiltModel,
				)
				from transformers.models.lilt.modeling_lilt import LILT_PRETRAINED_MODEL_ARCHIVE_LIST




class 							lowerCamelCase   :


	def __init__(    self				,     lowercase__				,     lowercase__=1_3				,     lowercase__=7				,     lowercase__=True				,     lowercase__=True				,     lowercase__=True				,     lowercase__=True				,     lowercase__=9_9				,     lowercase__=2_4				,     lowercase__=2				,     lowercase__=6				,     lowercase__=3_7				,     lowercase__="gelu"				,     lowercase__=0.1				,     lowercase__=0.1				,     lowercase__=5_1_2				,     lowercase__=1_6				,     lowercase__=2				,     lowercase__=0.0_2				,     lowercase__=3				,     lowercase__=None				,     lowercase__=1_0_0_0				,     ):
							__UpperCAmelCase			:							int	    =	parent
							__UpperCAmelCase			:							Optional[int]	    =	batch_size
							__UpperCAmelCase			:							Any	    =	seq_length
							__UpperCAmelCase			:							Dict	    =	is_training
							__UpperCAmelCase			:							str	    =	use_input_mask
							__UpperCAmelCase			:							Dict	    =	use_token_type_ids
							__UpperCAmelCase			:							Tuple	    =	use_labels
							__UpperCAmelCase			:							Union[str, Any]	    =	vocab_size
							__UpperCAmelCase			:							Tuple	    =	hidden_size
							__UpperCAmelCase			:							Dict	    =	num_hidden_layers
							__UpperCAmelCase			:							Optional[Any]	    =	num_attention_heads
							__UpperCAmelCase			:							Union[str, Any]	    =	intermediate_size
							__UpperCAmelCase			:							Optional[int]	    =	hidden_act
							__UpperCAmelCase			:							int	    =	hidden_dropout_prob
							__UpperCAmelCase			:							List[Any]	    =	attention_probs_dropout_prob
							__UpperCAmelCase			:							List[Any]	    =	max_position_embeddings
							__UpperCAmelCase			:							Optional[Any]	    =	type_vocab_size
							__UpperCAmelCase			:							Optional[Any]	    =	type_sequence_label_size
							__UpperCAmelCase			:							str	    =	initializer_range
							__UpperCAmelCase			:							str	    =	num_labels
							__UpperCAmelCase			:							str	    =	scope
							__UpperCAmelCase			:							Union[str, Any]	    =	range_bbox


	def  A(    self):
							__UpperCAmelCase			:							Union[str, Any]	    =	ids_tensor([self.batch_size, self.seq_length]				,     self.vocab_size)

							__UpperCAmelCase			:							Optional[int]	    =	ids_tensor([self.batch_size, self.seq_length, 4]				,     self.range_bbox)
							# Ensure that bbox is legal
							for i in range(bbox.shape[0]):
													for j in range(bbox.shape[1]):
																			if bbox[i, j, 3] < bbox[i, j, 1]:
																									__UpperCAmelCase			:							Optional[int]	    =	bbox[i, j, 3]
																									__UpperCAmelCase			:							str	    =	bbox[i, j, 1]
																									__UpperCAmelCase			:							Union[str, Any]	    =	t
																			if bbox[i, j, 2] < bbox[i, j, 0]:
																									__UpperCAmelCase			:							Optional[int]	    =	bbox[i, j, 2]
																									__UpperCAmelCase			:							int	    =	bbox[i, j, 0]
																									__UpperCAmelCase			:							Dict	    =	t

							__UpperCAmelCase			:							Tuple	    =	None
							if self.use_input_mask:
													__UpperCAmelCase			:							Any	    =	ids_tensor([self.batch_size, self.seq_length]				,     vocab_size=2)

							__UpperCAmelCase			:							List[Any]	    =	None
							if self.use_token_type_ids:
													__UpperCAmelCase			:							Tuple	    =	ids_tensor([self.batch_size, self.seq_length]				,     self.type_vocab_size)

							__UpperCAmelCase			:							List[str]	    =	None
							__UpperCAmelCase			:							Union[str, Any]	    =	None
							if self.use_labels:
													__UpperCAmelCase			:							Dict	    =	ids_tensor([self.batch_size]				,     self.type_sequence_label_size)
													__UpperCAmelCase			:							Optional[int]	    =	ids_tensor([self.batch_size, self.seq_length]				,     self.num_labels)

							__UpperCAmelCase			:							int	    =	self.get_config()

							return config, input_ids, bbox, token_type_ids, input_mask, sequence_labels, token_labels


	def  A(    self):
							return LiltConfig(
							    vocab_size=self.vocab_size				,     hidden_size=self.hidden_size				,     num_hidden_layers=self.num_hidden_layers				,     num_attention_heads=self.num_attention_heads				,     intermediate_size=self.intermediate_size				,     hidden_act=self.hidden_act				,     hidden_dropout_prob=self.hidden_dropout_prob				,     attention_probs_dropout_prob=self.attention_probs_dropout_prob				,     max_position_embeddings=self.max_position_embeddings				,     type_vocab_size=self.type_vocab_size				,     initializer_range=self.initializer_range				,     )


	def  A(    self				,     lowercase__				,     lowercase__				,     lowercase__				,     lowercase__				,     lowercase__				,     lowercase__				,     lowercase__				,     ):
							__UpperCAmelCase			:							List[Any]	    =	LiltModel(config=lowercase__)
							model.to(lowercase__)
							model.eval()
							__UpperCAmelCase			:							List[Any]	    =	model(lowercase__				,     bbox=lowercase__				,     attention_mask=lowercase__				,     token_type_ids=lowercase__)
							__UpperCAmelCase			:							Optional[int]	    =	model(lowercase__				,     bbox=lowercase__				,     token_type_ids=lowercase__)
							__UpperCAmelCase			:							Optional[int]	    =	model(lowercase__				,     bbox=lowercase__)

							self.parent.assertEqual(result.last_hidden_state.shape				,     (self.batch_size, self.seq_length, self.hidden_size))
							self.parent.assertEqual(result.pooler_output.shape				,     (self.batch_size, self.hidden_size))


	def  A(    self				,     lowercase__				,     lowercase__				,     lowercase__				,     lowercase__				,     lowercase__				,     lowercase__				,     lowercase__				,     ):
							__UpperCAmelCase			:							Optional[int]	    =	self.num_labels
							__UpperCAmelCase			:							Dict	    =	LiltForTokenClassification(config=lowercase__)
							model.to(lowercase__)
							model.eval()
							__UpperCAmelCase			:							Dict	    =	model(
							    lowercase__				,     bbox=lowercase__				,     attention_mask=lowercase__				,     token_type_ids=lowercase__				,     labels=lowercase__)
							self.parent.assertEqual(result.logits.shape				,     (self.batch_size, self.seq_length, self.num_labels))


	def  A(    self				,     lowercase__				,     lowercase__				,     lowercase__				,     lowercase__				,     lowercase__				,     lowercase__				,     lowercase__				,     ):
							__UpperCAmelCase			:							Dict	    =	LiltForQuestionAnswering(config=lowercase__)
							model.to(lowercase__)
							model.eval()
							__UpperCAmelCase			:							List[Any]	    =	model(
							    lowercase__				,     bbox=lowercase__				,     attention_mask=lowercase__				,     token_type_ids=lowercase__				,     start_positions=lowercase__				,     end_positions=lowercase__				,     )
							self.parent.assertEqual(result.start_logits.shape				,     (self.batch_size, self.seq_length))
							self.parent.assertEqual(result.end_logits.shape				,     (self.batch_size, self.seq_length))


	def  A(    self):
							__UpperCAmelCase			:							Union[str, Any]	    =	self.prepare_config_and_inputs()
							(
							    (
							    __UpperCAmelCase
							)    ,      (
							    __UpperCAmelCase
							)    ,      (
							    __UpperCAmelCase
							)    ,      (
							    __UpperCAmelCase
							)    ,      (
							    __UpperCAmelCase
							)    ,      (
							    __UpperCAmelCase
							)    ,      (
							    __UpperCAmelCase
							)    ,      
							)			:							Any	    =	config_and_inputs
							__UpperCAmelCase			:							int	    =	{
							    '''input_ids''': input_ids,
							    '''bbox''': bbox,
							    '''token_type_ids''': token_type_ids,
							    '''attention_mask''': input_mask,
							}
							return config, inputs_dict






@require_torch
class 							lowerCamelCase   (  _UpperCamelCase							,						_UpperCamelCase							,						_UpperCamelCase							,						unittest.TestCase ):
	_lowerCAmelCase     :		str										=						(
	    (
	        LiltModel,
	        LiltForSequenceClassification,
	        LiltForTokenClassification,
	        LiltForQuestionAnswering,
	    )
	    if is_torch_available()
	    else ()
	)
	_lowerCAmelCase     :		str										=						(
	    {
	        '''feature-extraction''': LiltModel,
	        '''question-answering''': LiltForQuestionAnswering,
	        '''text-classification''': LiltForSequenceClassification,
	        '''token-classification''': LiltForTokenClassification,
	        '''zero-shot''': LiltForSequenceClassification,
	    }
	    if is_torch_available()
	    else {}
	)
	_lowerCAmelCase     :		Any										=						False
	_lowerCAmelCase     :		List[Any]										=						False


	def  A(    self				,     lowercase__				,     lowercase__				,     lowercase__				,     lowercase__				,     lowercase__):
							return True


	def  A(    self):
							__UpperCAmelCase			:							Tuple	    =	LiltModelTester(self)
							__UpperCAmelCase			:							List[Any]	    =	ConfigTester(self				,     config_class=lowercase__				,     hidden_size=3_7)


	def  A(    self):
							self.config_tester.run_common_tests()


	def  A(    self):
							__UpperCAmelCase			:							Union[str, Any]	    =	self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_model(*lowercase__)


	def  A(    self):
							__UpperCAmelCase			:							int	    =	self.model_tester.prepare_config_and_inputs()
							for type in ["absolute", "relative_key", "relative_key_query"]:
													__UpperCAmelCase			:							Any	    =	type
													self.model_tester.create_and_check_model(*lowercase__)


	def  A(    self):
							__UpperCAmelCase			:							Dict	    =	self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_token_classification(*lowercase__)


	def  A(    self):
							__UpperCAmelCase			:							Optional[int]	    =	self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_question_answering(*lowercase__)


	@slow
	def  A(    self):
							for model_name in LILT_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
													__UpperCAmelCase			:							Optional[Any]	    =	LiltModel.from_pretrained(lowercase__)
													self.assertIsNotNone(lowercase__)






@require_torch
@slow
class 							lowerCamelCase   (  unittest.TestCase ):


	def  A(    self):
							__UpperCAmelCase			:							Any	    =	LiltModel.from_pretrained('''SCUT-DLVCLab/lilt-roberta-en-base''').to(lowercase__)

							__UpperCAmelCase			:							str	    =	torch.tensor([[1, 2]]				,     device=lowercase__)
							__UpperCAmelCase			:							List[str]	    =	torch.tensor([[[1, 2, 3, 4], [5, 6, 7, 8]]]				,     device=lowercase__)

							# forward pass
							with torch.no_grad():
													__UpperCAmelCase			:							Dict	    =	model(input_ids=lowercase__				,     bbox=lowercase__)

							__UpperCAmelCase			:							List[Any]	    =	torch.Size([1, 2, 7_6_8])
							__UpperCAmelCase			:							Union[str, Any]	    =	torch.tensor(
							    [[-0.0_6_5_3, 0.0_9_5_0, -0.0_0_6_1], [-0.0_5_4_5, 0.0_9_2_6, -0.0_3_2_4]]				,     device=lowercase__				,     )

							self.assertTrue(outputs.last_hidden_state.shape				,     lowercase__)
							self.assertTrue(torch.allclose(outputs.last_hidden_state[0, :, :3]				,     lowercase__				,     atol=1e-3))

