



from __future__ import annotations

import unittest

import numpy as np

from transformers import LayoutLMConfig, is_tf_available
from transformers.testing_utils import require_tf, slow

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
			import tensorflow as tf

			from transformers.models.layoutlm.modeling_tf_layoutlm import (
			    TF_LAYOUTLM_PRETRAINED_MODEL_ARCHIVE_LIST,
			    TFLayoutLMForMaskedLM,
			    TFLayoutLMForQuestionAnswering,
			    TFLayoutLMForSequenceClassification,
			    TFLayoutLMForTokenClassification,
			    TFLayoutLMModel,
			)


class 						lowercase    :
						'''simple docstring'''


						def __init__(  self    :				List[Any]       ,	snake_case    :				List[str]       ,	snake_case    :				List[str]=13       ,	snake_case    :				Optional[int]=7       ,	snake_case    :				Dict=True       ,	snake_case    :				Union[str, Any]=True       ,	snake_case    :				Dict=True       ,	snake_case    :				Optional[int]=True       ,	snake_case    :				Any=99       ,	snake_case    :				int=32       ,	snake_case    :				Union[str, Any]=2       ,	snake_case    :				Any=4       ,	snake_case    :				Any=37       ,	snake_case    :				List[Any]="gelu"       ,	snake_case    :				List[str]=0.1       ,	snake_case    :				List[str]=0.1       ,	snake_case    :				List[str]=512       ,	snake_case    :				Tuple=16       ,	snake_case    :				int=2       ,	snake_case    :				Dict=0.02       ,	snake_case    :				List[Any]=3       ,	snake_case    :				int=4       ,	snake_case    :				List[Any]=None       ,	snake_case    :				Tuple=1000       ,	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Tuple						=				parent
											SCREAMING_SNAKE_CASE							:     List[Any]						=				batch_size
											SCREAMING_SNAKE_CASE							:     List[Any]						=				seq_length
											SCREAMING_SNAKE_CASE							:     Tuple						=				is_training
											SCREAMING_SNAKE_CASE							:     Any						=				use_input_mask
											SCREAMING_SNAKE_CASE							:     Any						=				use_token_type_ids
											SCREAMING_SNAKE_CASE							:     Optional[int]						=				use_labels
											SCREAMING_SNAKE_CASE							:     Tuple						=				vocab_size
											SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				hidden_size
											SCREAMING_SNAKE_CASE							:     Tuple						=				num_hidden_layers
											SCREAMING_SNAKE_CASE							:     int						=				num_attention_heads
											SCREAMING_SNAKE_CASE							:     List[str]						=				intermediate_size
											SCREAMING_SNAKE_CASE							:     List[str]						=				hidden_act
											SCREAMING_SNAKE_CASE							:     str						=				hidden_dropout_prob
											SCREAMING_SNAKE_CASE							:     Dict						=				attention_probs_dropout_prob
											SCREAMING_SNAKE_CASE							:     Optional[int]						=				max_position_embeddings
											SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				type_vocab_size
											SCREAMING_SNAKE_CASE							:     Dict						=				type_sequence_label_size
											SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				initializer_range
											SCREAMING_SNAKE_CASE							:     Dict						=				num_labels
											SCREAMING_SNAKE_CASE							:     int						=				num_choices
											SCREAMING_SNAKE_CASE							:     List[Any]						=				scope
											SCREAMING_SNAKE_CASE							:     Dict						=				range_bbox







						def       lowerCamelCase_				(  self    :				Union[str, Any]	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     List[str]						=				ids_tensor([self.batch_size, self.seq_length]       ,	self.vocab_size	)

											# convert bbox to numpy since TF does not support item assignment
											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				ids_tensor([self.batch_size, self.seq_length, 4]       ,	self.range_bbox	).numpy()
											# Ensure that bbox is legal
											for i in range(bbox.shape[0]	):
																for j in range(bbox.shape[1]	):
																					if bbox[i, j, 3] < bbox[i, j, 1]:
																										SCREAMING_SNAKE_CASE							:     Any						=				bbox[i, j, 3]
																										SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				bbox[i, j, 1]
																										SCREAMING_SNAKE_CASE							:     List[str]						=				t
																					if bbox[i, j, 2] < bbox[i, j, 0]:
																										SCREAMING_SNAKE_CASE							:     int						=				bbox[i, j, 2]
																										SCREAMING_SNAKE_CASE							:     Dict						=				bbox[i, j, 0]
																										SCREAMING_SNAKE_CASE							:     Tuple						=				t
											SCREAMING_SNAKE_CASE							:     Optional[int]						=				tf.convert_to_tensor(snake_case	)

											SCREAMING_SNAKE_CASE							:     Tuple						=				None
											if self.use_input_mask:
																SCREAMING_SNAKE_CASE							:     Optional[int]						=				random_attention_mask([self.batch_size, self.seq_length]	)

											SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				None
											if self.use_token_type_ids:
																SCREAMING_SNAKE_CASE							:     Optional[int]						=				ids_tensor([self.batch_size, self.seq_length]       ,	self.type_vocab_size	)

											SCREAMING_SNAKE_CASE							:     Dict						=				None
											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				None
											SCREAMING_SNAKE_CASE							:     List[str]						=				None
											if self.use_labels:
																SCREAMING_SNAKE_CASE							:     List[Any]						=				ids_tensor([self.batch_size]       ,	self.type_sequence_label_size	)
																SCREAMING_SNAKE_CASE							:     int						=				ids_tensor([self.batch_size, self.seq_length]       ,	self.num_labels	)
																SCREAMING_SNAKE_CASE							:     Optional[int]						=				ids_tensor([self.batch_size]       ,	self.num_choices	)

											SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				LayoutLMConfig(
											    vocab_size=self.vocab_size       ,	hidden_size=self.hidden_size       ,	num_hidden_layers=self.num_hidden_layers       ,	num_attention_heads=self.num_attention_heads       ,	intermediate_size=self.intermediate_size       ,	hidden_act=self.hidden_act       ,	hidden_dropout_prob=self.hidden_dropout_prob       ,	attention_probs_dropout_prob=self.attention_probs_dropout_prob       ,	max_position_embeddings=self.max_position_embeddings       ,	type_vocab_size=self.type_vocab_size       ,	initializer_range=self.initializer_range       ,	)

											return config, input_ids, bbox, token_type_ids, input_mask, sequence_labels, token_labels, choice_labels







						def       lowerCamelCase_				(  self    :				Optional[Any]       ,	snake_case    :				Dict       ,	snake_case    :				List[Any]       ,	snake_case    :				int       ,	snake_case    :				Tuple       ,	snake_case    :				List[Any]       ,	snake_case    :				Tuple       ,	snake_case    :				List[str]       ,	snake_case    :				Union[str, Any]	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     str						=				TFLayoutLMModel(config=snake_case	)

											SCREAMING_SNAKE_CASE							:     Tuple						=				model(snake_case       ,	snake_case       ,	attention_mask=snake_case       ,	token_type_ids=snake_case	)
											SCREAMING_SNAKE_CASE							:     Tuple						=				model(snake_case       ,	snake_case       ,	token_type_ids=snake_case	)
											SCREAMING_SNAKE_CASE							:     List[Any]						=				model(snake_case       ,	snake_case	)
											self.parent.assertEqual(result.last_hidden_state.shape       ,	(self.batch_size, self.seq_length, self.hidden_size)	)
											self.parent.assertEqual(result.pooler_output.shape       ,	(self.batch_size, self.hidden_size)	)







						def       lowerCamelCase_				(  self    :				Optional[int]       ,	snake_case    :				List[str]       ,	snake_case    :				Tuple       ,	snake_case    :				List[Any]       ,	snake_case    :				Union[str, Any]       ,	snake_case    :				List[str]       ,	snake_case    :				Union[str, Any]       ,	snake_case    :				Union[str, Any]       ,	snake_case    :				Any	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Tuple						=				TFLayoutLMForMaskedLM(config=snake_case	)

											SCREAMING_SNAKE_CASE							:     Optional[int]						=				model(snake_case       ,	snake_case       ,	attention_mask=snake_case       ,	token_type_ids=snake_case       ,	labels=snake_case	)
											self.parent.assertEqual(result.logits.shape       ,	(self.batch_size, self.seq_length, self.vocab_size)	)







						def       lowerCamelCase_				(  self    :				List[str]       ,	snake_case    :				Optional[Any]       ,	snake_case    :				str       ,	snake_case    :				List[str]       ,	snake_case    :				List[Any]       ,	snake_case    :				Union[str, Any]       ,	snake_case    :				int       ,	snake_case    :				List[str]       ,	snake_case    :				Any	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     List[Any]						=				self.num_labels
											SCREAMING_SNAKE_CASE							:     Tuple						=				TFLayoutLMForSequenceClassification(config=snake_case	)

											SCREAMING_SNAKE_CASE							:     int						=				model(snake_case       ,	snake_case       ,	attention_mask=snake_case       ,	token_type_ids=snake_case	)
											self.parent.assertEqual(result.logits.shape       ,	(self.batch_size, self.num_labels)	)







						def       lowerCamelCase_				(  self    :				Any       ,	snake_case    :				Dict       ,	snake_case    :				Union[str, Any]       ,	snake_case    :				Dict       ,	snake_case    :				Any       ,	snake_case    :				Union[str, Any]       ,	snake_case    :				Any       ,	snake_case    :				Optional[int]       ,	snake_case    :				str	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				self.num_labels
											SCREAMING_SNAKE_CASE							:     Any						=				TFLayoutLMForTokenClassification(config=snake_case	)

											SCREAMING_SNAKE_CASE							:     str						=				model(snake_case       ,	snake_case       ,	attention_mask=snake_case       ,	token_type_ids=snake_case       ,	labels=snake_case	)
											self.parent.assertEqual(result.logits.shape       ,	(self.batch_size, self.seq_length, self.num_labels)	)







						def       lowerCamelCase_				(  self    :				Union[str, Any]       ,	snake_case    :				int       ,	snake_case    :				Union[str, Any]       ,	snake_case    :				List[Any]       ,	snake_case    :				List[Any]       ,	snake_case    :				Union[str, Any]       ,	snake_case    :				Optional[int]       ,	snake_case    :				Union[str, Any]       ,	snake_case    :				str	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Optional[int]						=				TFLayoutLMForQuestionAnswering(config=snake_case	)

											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				model(snake_case       ,	snake_case       ,	attention_mask=snake_case       ,	token_type_ids=snake_case	)
											self.parent.assertEqual(result.start_logits.shape       ,	(self.batch_size, self.seq_length)	)
											self.parent.assertEqual(result.end_logits.shape       ,	(self.batch_size, self.seq_length)	)







						def       lowerCamelCase_				(  self    :				Dict	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     int						=				self.prepare_config_and_inputs()
											(
											    (
											    SCREAMING_SNAKE_CASE
											)					,  (
											    SCREAMING_SNAKE_CASE
											)					,  (
											    SCREAMING_SNAKE_CASE
											)					,  (
											    SCREAMING_SNAKE_CASE
											)					,  (
											    SCREAMING_SNAKE_CASE
											)					,  (
											    SCREAMING_SNAKE_CASE
											)					,  (
											    SCREAMING_SNAKE_CASE
											)					,  (
											    SCREAMING_SNAKE_CASE
											)					,  
											)							:     Optional[Any]						=				config_and_inputs
											SCREAMING_SNAKE_CASE							:     List[str]						=				{
											    'input_ids': input_ids,
											    'bbox': bbox,
											    'token_type_ids': token_type_ids,
											    'attention_mask': input_mask,
											}
											return config, inputs_dict


@require_tf
class 						lowercase    (							SCREAMING_SNAKE_CASE_      ,    SCREAMING_SNAKE_CASE_      ,    unittest.TestCase):
						'''simple docstring'''


						UpperCAmelCase			:		Any		=		(
						    (
						        TFLayoutLMModel,
						        TFLayoutLMForMaskedLM,
						        TFLayoutLMForTokenClassification,
						        TFLayoutLMForSequenceClassification,
						        TFLayoutLMForQuestionAnswering,
						    )
						    if is_tf_available()
						    else ()
						)
						UpperCAmelCase			:		Union[str, Any]		=		(
						    {
						        'feature-extraction': TFLayoutLMModel,
						        'fill-mask': TFLayoutLMForMaskedLM,
						        'text-classification': TFLayoutLMForSequenceClassification,
						        'token-classification': TFLayoutLMForTokenClassification,
						        'zero-shot': TFLayoutLMForSequenceClassification,
						    }
						    if is_tf_available()
						    else {}
						)
						UpperCAmelCase			:		Any		=		False
						UpperCAmelCase			:		Tuple		=		True
						UpperCAmelCase			:		List[Any]		=		10







						def       lowerCamelCase_				(  self    :				Optional[Any]	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     List[str]						=				TFLayoutLMModelTester(self	)
											SCREAMING_SNAKE_CASE							:     int						=				ConfigTester(self       ,	config_class=snake_case       ,	hidden_size=37	)







						def       lowerCamelCase_				(  self    :				Optional[int]	):



											'''simple docstring'''


											self.config_tester.run_common_tests()







						def       lowerCamelCase_				(  self    :				Union[str, Any]	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     List[Any]						=				self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_model(*snake_case	)







						def       lowerCamelCase_				(  self    :				str	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_masked_lm(*snake_case	)







						def       lowerCamelCase_				(  self    :				int	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_sequence_classification(*snake_case	)







						def       lowerCamelCase_				(  self    :				Optional[Any]	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     List[str]						=				self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_token_classification(*snake_case	)







						def       lowerCamelCase_				(  self    :				List[str]	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Optional[int]						=				self.model_tester.prepare_config_and_inputs()
											self.model_tester.create_and_check_for_question_answering(*snake_case	)







						@slow
						def       lowerCamelCase_				(  self    :				Optional[Any]	):



											'''simple docstring'''


											for model_name in TF_LAYOUTLM_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																SCREAMING_SNAKE_CASE							:     Tuple						=				TFLayoutLMModel.from_pretrained(snake_case	)
																self.assertIsNotNone(snake_case	)







						@unittest.skip('Onnx compliancy broke with TF 2.10'	)
						def       lowerCamelCase_				(  self    :				Tuple	):



											'''simple docstring'''


											pass


def        __a			(   )     ->     Any:
					# Here we prepare a batch of 2 sequences to test a LayoutLM forward pass on:
					# fmt: off
					SCREAMING_SNAKE_CASE							:     Optional[Any]						=				tf.convert_to_tensor([[101,1019,1014,1016,1037,1_2849,4747,1004,1_4246,2278,5439,4524,5002,2930,2193,2930,4341,3208,1005,1055,2171,2848,1_1300,3531,102],[101,4070,4034,7020,1024,3058,1015,1013,2861,1013,6070,1_9274,2772,6205,2_7814,1_6147,1_6147,4343,2047,1_0283,1_0969,1_4389,1012,2338,102]]   )  # noqa: E231
					SCREAMING_SNAKE_CASE							:     Optional[Any]						=				tf.convert_to_tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1],]   )  # noqa: E231
					SCREAMING_SNAKE_CASE							:     int						=				tf.convert_to_tensor([[[0,0,0,0],[423,237,440,251],[427,272,441,287],[419,115,437,129],[961,885,992,912],[256,38,330,58],[256,38,330,58],[336,42,353,57],[360,39,401,56],[360,39,401,56],[411,39,471,59],[479,41,528,59],[533,39,630,60],[67,113,134,131],[141,115,209,132],[68,149,133,166],[141,149,187,164],[195,148,287,165],[195,148,287,165],[195,148,287,165],[295,148,349,165],[441,149,492,166],[497,149,546,164],[64,201,125,218],[1000,1000,1000,1000]],[[0,0,0,0],[662,150,754,166],[665,199,742,211],[519,213,554,228],[519,213,554,228],[134,433,187,454],[130,467,204,480],[130,467,204,480],[130,467,204,480],[130,467,204,480],[130,467,204,480],[314,469,376,482],[504,684,582,706],[941,825,973,900],[941,825,973,900],[941,825,973,900],[941,825,973,900],[610,749,652,765],[130,659,168,672],[176,657,237,672],[238,657,312,672],[443,653,628,672],[443,653,628,672],[716,301,825,317],[1000,1000,1000,1000]]]   )  # noqa: E231
					SCREAMING_SNAKE_CASE							:     int						=				tf.convert_to_tensor([[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]]   )  # noqa: E231
					# these are sequence labels (i.e. at the token level)
					SCREAMING_SNAKE_CASE							:     List[Any]						=				tf.convert_to_tensor([[-100,10,10,10,9,1,-100,7,7,-100,7,7,4,2,5,2,8,8,-100,-100,5,0,3,2,-100],[-100,12,12,12,-100,12,10,-100,-100,-100,-100,10,12,9,-100,-100,-100,10,10,10,9,12,-100,10,-100]]   )  # noqa: E231
					# fmt: on

					return input_ids, attention_mask, bbox, token_type_ids, labels



@require_tf
class 						lowercase    (							unittest.TestCase):
						'''simple docstring'''


						@slow
						def       lowerCamelCase_				(  self    :				Optional[Any]	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     str						=				TFLayoutLMModel.from_pretrained('microsoft/layoutlm-base-uncased'	)

											SCREAMING_SNAKE_CASE					,  SCREAMING_SNAKE_CASE					,  SCREAMING_SNAKE_CASE					,  SCREAMING_SNAKE_CASE					,  SCREAMING_SNAKE_CASE							:     Tuple						=				prepare_layoutlm_batch_inputs()

											# forward pass
											SCREAMING_SNAKE_CASE							:     List[str]						=				model(input_ids=snake_case       ,	bbox=snake_case       ,	attention_mask=snake_case       ,	token_type_ids=snake_case	)

											# test the sequence output on [0, :3, :3]
											SCREAMING_SNAKE_CASE							:     str						=				tf.convert_to_tensor(
											    [[0.1785, -0.1947, -0.0425], [-0.3254, -0.2807, 0.2553], [-0.5391, -0.3322, 0.3364]]       ,	)

											self.assertTrue(np.allclose(outputs.last_hidden_state[0, :3, :3]       ,	snake_case       ,	atol=1E-3	)	)

											# test the pooled output on [1, :3]
											SCREAMING_SNAKE_CASE							:     int						=				tf.convert_to_tensor([-0.6580, -0.0214, 0.8552]	)

											self.assertTrue(np.allclose(outputs.pooler_output[1, :3]       ,	snake_case       ,	atol=1E-3	)	)







						@slow
						def       lowerCamelCase_				(  self    :				List[str]	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				TFLayoutLMForSequenceClassification.from_pretrained('microsoft/layoutlm-base-uncased'       ,	num_labels=2	)

											SCREAMING_SNAKE_CASE					,  SCREAMING_SNAKE_CASE					,  SCREAMING_SNAKE_CASE					,  SCREAMING_SNAKE_CASE					,  SCREAMING_SNAKE_CASE							:     List[Any]						=				prepare_layoutlm_batch_inputs()

											# forward pass
											SCREAMING_SNAKE_CASE							:     Tuple						=				model(
											    input_ids=snake_case       ,	bbox=snake_case       ,	attention_mask=snake_case       ,	token_type_ids=snake_case       ,	labels=tf.convert_to_tensor([1, 1]	)       ,	)

											# test whether we get a loss as a scalar
											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				outputs.loss
											SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				(2,)
											self.assertEqual(loss.shape       ,	snake_case	)

											# test the shape of the logits
											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				outputs.logits
											SCREAMING_SNAKE_CASE							:     Optional[int]						=				(2, 2)
											self.assertEqual(logits.shape       ,	snake_case	)







						@slow
						def       lowerCamelCase_				(  self    :				int	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Optional[int]						=				TFLayoutLMForTokenClassification.from_pretrained('microsoft/layoutlm-base-uncased'       ,	num_labels=13	)

											SCREAMING_SNAKE_CASE					,  SCREAMING_SNAKE_CASE					,  SCREAMING_SNAKE_CASE					,  SCREAMING_SNAKE_CASE					,  SCREAMING_SNAKE_CASE							:     List[str]						=				prepare_layoutlm_batch_inputs()

											# forward pass
											SCREAMING_SNAKE_CASE							:     Tuple						=				model(
											    input_ids=snake_case       ,	bbox=snake_case       ,	attention_mask=snake_case       ,	token_type_ids=snake_case       ,	labels=snake_case	)

											# test the shape of the logits
											SCREAMING_SNAKE_CASE							:     Any						=				outputs.logits
											SCREAMING_SNAKE_CASE							:     str						=				tf.convert_to_tensor((2, 25, 13)	)
											self.assertEqual(logits.shape       ,	snake_case	)







						@slow
						def       lowerCamelCase_				(  self    :				Dict	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     int						=				TFLayoutLMForQuestionAnswering.from_pretrained('microsoft/layoutlm-base-uncased'	)

											SCREAMING_SNAKE_CASE					,  SCREAMING_SNAKE_CASE					,  SCREAMING_SNAKE_CASE					,  SCREAMING_SNAKE_CASE					,  SCREAMING_SNAKE_CASE							:     List[str]						=				prepare_layoutlm_batch_inputs()

											# forward pass
											SCREAMING_SNAKE_CASE							:     List[str]						=				model(input_ids=snake_case       ,	bbox=snake_case       ,	attention_mask=snake_case       ,	token_type_ids=snake_case	)

											# test the shape of the logits
											SCREAMING_SNAKE_CASE							:     str						=				tf.convert_to_tensor((2, 25)	)
											self.assertEqual(outputs.start_logits.shape       ,	snake_case	)
											self.assertEqual(outputs.end_logits.shape       ,	snake_case	)




import json
import os
from functools import lru_cache
from typing import TYPE_CHECKING, List, Optional, Tuple

import regex as re

from ...tokenization_utils import AddedToken, PreTrainedTokenizer
from ...utils import logging


if TYPE_CHECKING:
			from transformers.pipelines.conversational import Conversation

_lowerCamelCase    :		Any		      =    logging.get_logger(__name__)


_lowerCamelCase    :		Union[str, Any]		      =    {
    """vocab_file""": """vocab.json""",
    """merges_file""": """merges.txt""",
    """tokenizer_config_file""": """tokenizer_config.json""",
}

_lowerCamelCase    :		Optional[Any]		      =    {
    """vocab_file""": {"""facebook/blenderbot-3B""": """https://huggingface.co/facebook/blenderbot-3B/resolve/main/vocab.json"""},
    """merges_file""": {"""facebook/blenderbot-3B""": """https://huggingface.co/facebook/blenderbot-3B/resolve/main/merges.txt"""},
    """tokenizer_config_file""": {
        """facebook/blenderbot-3B""": """https://huggingface.co/facebook/blenderbot-3B/resolve/main/tokenizer_config.json"""
    },
}

_lowerCamelCase    :		Optional[int]		      =    {"""facebook/blenderbot-3B""": 128}

@lru_cache()
# Copied from transformers.models.roberta.tokenization_roberta.bytes_to_unicode
def        __a			(   )     ->     int:
					SCREAMING_SNAKE_CASE							:     Dict						=				(
					    list(range(ord('!'   )      ,	ord('~'   ) + 1   )   ) + list(range(ord('¡'   )      ,	ord('¬'   ) + 1   )   ) + list(range(ord('®'   )      ,	ord('ÿ'   ) + 1   )   )
					)
					SCREAMING_SNAKE_CASE							:     Optional[int]						=				bs[:]
					SCREAMING_SNAKE_CASE							:     int						=				0
					for b in range(2**8   ):
										if b not in bs:
															bs.append(__lowerCAmelCase   )
															cs.append(2**8 + n   )
															n += 1
					SCREAMING_SNAKE_CASE							:     Any						=				[chr(__lowerCAmelCase   ) for n in cs]
					return dict(zip(__lowerCAmelCase      ,	__lowerCAmelCase   )   )

def        __a			(   __lowerCAmelCase   )     ->     Optional[Any]:
					SCREAMING_SNAKE_CASE							:     Optional[Any]						=				set()
					SCREAMING_SNAKE_CASE							:     List[str]						=				word[0]
					for char in word[1:]:
										pairs.add((prev_char, char)   )
										SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				char
					return pairs



class 						lowercase    (							SCREAMING_SNAKE_CASE_):
						'''simple docstring'''


						UpperCAmelCase			:		Optional[int]		=		VOCAB_FILES_NAMES
						UpperCAmelCase			:		Any		=		PRETRAINED_VOCAB_FILES_MAP
						UpperCAmelCase			:		str		=		PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
						UpperCAmelCase			:		Dict		=		['input_ids', 'attention_mask']







						def __init__(  self    :				Dict       ,	snake_case    :				Optional[int]       ,	snake_case    :				Tuple       ,	snake_case    :				Dict="replace"       ,	snake_case    :				Optional[Any]="<s>"       ,	snake_case    :				Dict="</s>"       ,	snake_case    :				str="</s>"       ,	snake_case    :				Tuple="<s>"       ,	snake_case    :				List[Any]="<unk>"       ,	snake_case    :				Dict="<pad>"       ,	snake_case    :				int="<mask>"       ,	snake_case    :				Any=False       ,	**snake_case    :				str       ,	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     int						=				AddedToken(snake_case       ,	lstrip=snake_case       ,	rstrip=snake_case	) if isinstance(snake_case       ,	snake_case	) else bos_token
											SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				AddedToken(snake_case       ,	lstrip=snake_case       ,	rstrip=snake_case	) if isinstance(snake_case       ,	snake_case	) else eos_token
											SCREAMING_SNAKE_CASE							:     int						=				AddedToken(snake_case       ,	lstrip=snake_case       ,	rstrip=snake_case	) if isinstance(snake_case       ,	snake_case	) else sep_token
											SCREAMING_SNAKE_CASE							:     Dict						=				AddedToken(snake_case       ,	lstrip=snake_case       ,	rstrip=snake_case	) if isinstance(snake_case       ,	snake_case	) else cls_token
											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				AddedToken(snake_case       ,	lstrip=snake_case       ,	rstrip=snake_case	) if isinstance(snake_case       ,	snake_case	) else unk_token
											SCREAMING_SNAKE_CASE							:     List[Any]						=				AddedToken(snake_case       ,	lstrip=snake_case       ,	rstrip=snake_case	) if isinstance(snake_case       ,	snake_case	) else pad_token

											# Mask token behave like a normal word, i.e. include the space before it
											SCREAMING_SNAKE_CASE							:     str						=				AddedToken(snake_case       ,	lstrip=snake_case       ,	rstrip=snake_case	) if isinstance(snake_case       ,	snake_case	) else mask_token

											super().__init__(
											    errors=snake_case       ,	bos_token=snake_case       ,	eos_token=snake_case       ,	unk_token=snake_case       ,	sep_token=snake_case       ,	cls_token=snake_case       ,	pad_token=snake_case       ,	mask_token=snake_case       ,	add_prefix_space=snake_case       ,	**snake_case       ,	)

											with open(snake_case       ,	encoding='utf-8'	) as vocab_handle:
																SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				json.load(snake_case	)
											SCREAMING_SNAKE_CASE							:     int						=				{v: k for k, v in self.encoder.items()}
											SCREAMING_SNAKE_CASE							:     Optional[int]						=				errors  # how to handle errors in decoding
											SCREAMING_SNAKE_CASE							:     List[Any]						=				bytes_to_unicode()
											SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				{v: k for k, v in self.byte_encoder.items()}
											with open(snake_case       ,	encoding='utf-8'	) as merges_handle:
																SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				merges_handle.read().split('\n'	)[1:-1]
											SCREAMING_SNAKE_CASE							:     str						=				[tuple(merge.split()	) for merge in bpe_merges]
											SCREAMING_SNAKE_CASE							:     List[str]						=				dict(zip(snake_case       ,	range(len(snake_case	)	)	)	)
											SCREAMING_SNAKE_CASE							:     int						=				{}
											SCREAMING_SNAKE_CASE							:     str						=				add_prefix_space

											# Should have added re.IGNORECASE so BPE merges can happen for capitalized versions of contractions
											SCREAMING_SNAKE_CASE							:     List[str]						=				re.compile(r'\'s|\'t|\'re|\'ve|\'m|\'ll|\'d| ?\p{L}+| ?\p{N}+| ?[^\s\p{L}\p{N}]+|\s+(?!\S)|\s+'	)







						@property
						# Copied from transformers.models.roberta.tokenization_roberta.RobertaTokenizer.vocab_size with Roberta->Blenderbot, RoBERTa->Blenderbot
						def       lowerCamelCase_				(  self    :				str	):



											'''simple docstring'''


											return len(self.encoder	)







						def       lowerCamelCase_				(  self    :				Optional[Any]	):



											'''simple docstring'''


											return dict(self.encoder       ,	**self.added_tokens_encoder	)







						def       lowerCamelCase_				(  self    :				List[str]       ,	snake_case    :				int	):



											'''simple docstring'''


											if token in self.cache:
																return self.cache[token]
											SCREAMING_SNAKE_CASE							:     int						=				tuple(snake_case	)
											SCREAMING_SNAKE_CASE							:     Dict						=				get_pairs(snake_case	)

											if not pairs:
																return token

											while True:
																SCREAMING_SNAKE_CASE							:     List[str]						=				min(snake_case       ,	key=lambda snake_case	: self.bpe_ranks.get(snake_case       ,	float('inf'	)	)	)
																if bigram not in self.bpe_ranks:
																					break
																SCREAMING_SNAKE_CASE					,  SCREAMING_SNAKE_CASE							:     Dict						=				bigram
																SCREAMING_SNAKE_CASE							:     Optional[int]						=				[]
																SCREAMING_SNAKE_CASE							:     Tuple						=				0
																while i < len(snake_case	):
																					try:
																										SCREAMING_SNAKE_CASE							:     List[str]						=				word.index(snake_case       ,	snake_case	)
																					except ValueError:
																										new_word.extend(word[i:]	)
																										break
																					else:
																										new_word.extend(word[i:j]	)
																										SCREAMING_SNAKE_CASE							:     Dict						=				j

																					if word[i] == first and i < len(snake_case	) - 1 and word[i + 1] == second:
																										new_word.append(first + second	)
																										i += 2
																					else:
																										new_word.append(word[i]	)
																										i += 1
																SCREAMING_SNAKE_CASE							:     Any						=				tuple(snake_case	)
																SCREAMING_SNAKE_CASE							:     Tuple						=				new_word
																if len(snake_case	) == 1:
																					break
																else:
																					SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				get_pairs(snake_case	)
											SCREAMING_SNAKE_CASE							:     List[Any]						=				' '.join(snake_case	)
											SCREAMING_SNAKE_CASE							:     Dict						=				word
											return word







						def       lowerCamelCase_				(  self    :				Any       ,	snake_case    :				Tuple	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Tuple						=				[]
											for token in re.findall(self.pat       ,	snake_case	):
																SCREAMING_SNAKE_CASE							:     Any						=				''.join(
																    self.byte_encoder[b] for b in token.encode('utf-8'	)	)  # Maps all our bytes to unicode strings, avoiding control tokens of the BPE (spaces in our case)
																bpe_tokens.extend(bpe_token for bpe_token in self.bpe(snake_case	).split(' '	)	)
											return bpe_tokens







						def       lowerCamelCase_				(  self    :				Any       ,	snake_case    :				int	):



											'''simple docstring'''


											return self.encoder.get(snake_case       ,	self.encoder.get(self.unk_token	)	)







						def       lowerCamelCase_				(  self    :				List[Any]       ,	snake_case    :				Dict	):



											'''simple docstring'''


											return self.decoder.get(snake_case	)







						def       lowerCamelCase_				(  self    :				List[Any]       ,	snake_case    :				int	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     List[Any]						=				''.join(snake_case	)
											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				bytearray([self.byte_decoder[c] for c in text]	).decode('utf-8'       ,	errors=self.errors	)
											return text







						def       lowerCamelCase_				(  self    :				Union[str, Any]       ,	snake_case    :				str       ,	snake_case    :				Optional[str] = None	):



											'''simple docstring'''


											if not os.path.isdir(snake_case	):
																logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''	)
																return
											SCREAMING_SNAKE_CASE							:     List[Any]						=				os.path.join(
											    snake_case       ,	(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']	)
											SCREAMING_SNAKE_CASE							:     List[str]						=				os.path.join(
											    snake_case       ,	(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['merges_file']	)

											with open(snake_case       ,	'w'       ,	encoding='utf-8'	) as f:
																f.write(json.dumps(self.encoder       ,	indent=2       ,	sort_keys=snake_case       ,	ensure_ascii=snake_case	) + '\n'	)

											SCREAMING_SNAKE_CASE							:     str						=				0
											with open(snake_case       ,	'w'       ,	encoding='utf-8'	) as writer:
																writer.write('#version: 0.2\n'	)
																for bpe_tokens, token_index in sorted(self.bpe_ranks.items()       ,	key=lambda snake_case	: kv[1]	):
																					if index != token_index:
																										logger.warning(
																										    f'''Saving vocabulary to {merge_file}: BPE merge indices are not consecutive.'''
																										    ' Please check that the tokenizer is not corrupted!'	)
																										SCREAMING_SNAKE_CASE							:     List[str]						=				token_index
																					writer.write(' '.join(snake_case	) + '\n'	)
																					index += 1

											return vocab_file, merge_file







						def       lowerCamelCase_				(  self    :				Dict       ,	snake_case    :				List[int]       ,	snake_case    :				Optional[List[int]] = None       ,	snake_case    :				bool = False	):



											'''simple docstring'''


											if already_has_special_tokens:
																return super().get_special_tokens_mask(
																    token_ids_a=snake_case       ,	token_ids_a=snake_case       ,	already_has_special_tokens=snake_case	)

											if token_ids_a is None:
																return [1] + ([0] * len(snake_case	)) + [1]
											return [1] + ([0] * len(snake_case	)) + [1, 1] + ([0] * len(snake_case	)) + [1]







						def       lowerCamelCase_				(  self    :				int       ,	snake_case    :				List[int]       ,	snake_case    :				Optional[List[int]] = None	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Dict						=				[self.sep_token_id]
											SCREAMING_SNAKE_CASE							:     int						=				[self.cls_token_id]

											if token_ids_a is None:
																return len(cls + token_ids_a + sep	) * [0]
											return len(cls + token_ids_a + sep + sep + token_ids_a + sep	) * [0]







						def       lowerCamelCase_				(  self    :				Optional[Any]       ,	snake_case    :				List[Any]       ,	snake_case    :				Optional[Any]=False       ,	**snake_case    :				Optional[int]	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Optional[int]						=				kwargs.pop('add_prefix_space'       ,	self.add_prefix_space	)
											if (is_split_into_words or add_prefix_space) and (len(snake_case	) > 0 and not text[0].isspace()):
																SCREAMING_SNAKE_CASE							:     Union[str, Any]						=				' ' + text
											return (text, kwargs)







						def       lowerCamelCase_				(  self    :				List[str]       ,	snake_case    :				List[int]       ,	snake_case    :				Optional[List[int]] = None	):



											'''simple docstring'''


											return token_ids_a + [self.eos_token_id]







						def       lowerCamelCase_				(  self    :				List[str]       ,	snake_case    :				"Conversation"	):



											'''simple docstring'''


											SCREAMING_SNAKE_CASE							:     Tuple						=				[]
											for is_user, text in conversation.iter_texts():
																if is_user:
																					# We need to space prefix as it's being done within blenderbot
																					inputs.append(' ' + text	)
																else:
																					# Generated responses should contain them already.
																					inputs.append(snake_case	)

											SCREAMING_SNAKE_CASE							:     Optional[Any]						=				'  '.join(snake_case	)
											SCREAMING_SNAKE_CASE							:     str						=				self.encode(snake_case	)
											if len(snake_case	) > self.model_max_length:
																SCREAMING_SNAKE_CASE							:     Optional[Any]						=				input_ids[-self.model_max_length :]
																logger.warning(f'''Trimmed input from conversation as it was longer than {self.model_max_length} tokens.'''	)
											return input_ids