






"""simple docstring"""







from typing import TYPE_CHECKING

from ...utils import (
    OptionalDependencyNotAvailable,
    _LazyModule,
    is_flax_available,
    is_tf_available,
    is_tokenizers_available,
    is_torch_available,
)


__a       :	str             = {
    'configuration_roberta': ['ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP', 'RobertaConfig', 'RobertaOnnxConfig'],
    'tokenization_roberta': ['RobertaTokenizer'],
}

try:
	if not is_tokenizers_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	__a       :	List[Any]             = ['RobertaTokenizerFast']

try:
	if not is_torch_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	__a       :	Dict             = [
	    'ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST',
	    'RobertaForCausalLM',
	    'RobertaForMaskedLM',
	    'RobertaForMultipleChoice',
	    'RobertaForQuestionAnswering',
	    'RobertaForSequenceClassification',
	    'RobertaForTokenClassification',
	    'RobertaModel',
	    'RobertaPreTrainedModel',
	]

try:
	if not is_tf_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	__a       :	str             = [
	    'TF_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST',
	    'TFRobertaForCausalLM',
	    'TFRobertaForMaskedLM',
	    'TFRobertaForMultipleChoice',
	    'TFRobertaForQuestionAnswering',
	    'TFRobertaForSequenceClassification',
	    'TFRobertaForTokenClassification',
	    'TFRobertaMainLayer',
	    'TFRobertaModel',
	    'TFRobertaPreTrainedModel',
	]

try:
	if not is_flax_available():
		raise OptionalDependencyNotAvailable()
except OptionalDependencyNotAvailable:
	pass
else:
	__a       :	Optional[Any]             = [
	    'FlaxRobertaForCausalLM',
	    'FlaxRobertaForMaskedLM',
	    'FlaxRobertaForMultipleChoice',
	    'FlaxRobertaForQuestionAnswering',
	    'FlaxRobertaForSequenceClassification',
	    'FlaxRobertaForTokenClassification',
	    'FlaxRobertaModel',
	    'FlaxRobertaPreTrainedModel',
	]


if TYPE_CHECKING:
	from .configuration_roberta import ROBERTA_PRETRAINED_CONFIG_ARCHIVE_MAP, RobertaConfig, RobertaOnnxConfig
	from .tokenization_roberta import RobertaTokenizer

	try:
		if not is_tokenizers_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .tokenization_roberta_fast import RobertaTokenizerFast

	try:
		if not is_torch_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_roberta import (
		    ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
		    RobertaForCausalLM,
		    RobertaForMaskedLM,
		    RobertaForMultipleChoice,
		    RobertaForQuestionAnswering,
		    RobertaForSequenceClassification,
		    RobertaForTokenClassification,
		    RobertaModel,
		    RobertaPreTrainedModel,
		)

	try:
		if not is_tf_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_tf_roberta import (
		    TF_ROBERTA_PRETRAINED_MODEL_ARCHIVE_LIST,
		    TFRobertaForCausalLM,
		    TFRobertaForMaskedLM,
		    TFRobertaForMultipleChoice,
		    TFRobertaForQuestionAnswering,
		    TFRobertaForSequenceClassification,
		    TFRobertaForTokenClassification,
		    TFRobertaMainLayer,
		    TFRobertaModel,
		    TFRobertaPreTrainedModel,
		)

	try:
		if not is_flax_available():
			raise OptionalDependencyNotAvailable()
	except OptionalDependencyNotAvailable:
		pass
	else:
		from .modeling_flax_roberta import (
		    FlaxRobertaForCausalLM,
		    FlaxRobertaForMaskedLM,
		    FlaxRobertaForMultipleChoice,
		    FlaxRobertaForQuestionAnswering,
		    FlaxRobertaForSequenceClassification,
		    FlaxRobertaForTokenClassification,
		    FlaxRobertaModel,
		    FlaxRobertaPreTrainedModel,
		)

else:
	import sys

	__a       :	Union[str, Any]             = _LazyModule(__name__, globals()['__file__'], _import_structure, module_spec=__spec__)










"""simple docstring"""







from __future__ import annotations

import copy
import inspect
import unittest

import numpy as np

from transformers import is_tf_available, is_vision_available
from transformers.models.auto import get_values
from transformers.testing_utils import require_tf, slow
from transformers.utils import cached_property

from ...test_configuration_common import ConfigTester
from ...test_modeling_tf_common import TFModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_tf_available():
	import tensorflow as tf

	from transformers import (
	    TF_LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST,
	    TF_MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
	    TF_MODEL_FOR_QUESTION_ANSWERING_MAPPING,
	    TF_MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
	    TF_MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
	    LayoutLMvaConfig,
	    TFLayoutLMvaForQuestionAnswering,
	    TFLayoutLMvaForSequenceClassification,
	    TFLayoutLMvaForTokenClassification,
	    TFLayoutLMvaModel,
	)

if is_vision_available():
	from PIL import Image

	from transformers import LayoutLMvaImageProcessor







class        _SCREAMING_SNAKE_CASE						:





							"""simple docstring"""





							def __init__(   self:     List[str]				,		__A:     str				,		__A:     Any=2				,		__A:     str=3				,		__A:     Tuple=4				,		__A:     Dict=2				,		__A:     List[Any]=7				,		__A:     Any=True				,		__A:     Any=True				,		__A:     List[str]=True				,		__A:     Optional[int]=True				,		__A:     Optional[int]=99				,		__A:     Tuple=36				,		__A:     List[str]=2				,		__A:     Dict=4				,		__A:     List[str]=37				,		__A:     Optional[int]="gelu"				,		__A:     Optional[int]=0.1				,		__A:     Tuple=0.1				,		__A:     List[Any]=512				,		__A:     List[str]=16				,		__A:     Any=2				,		__A:     Union[str, Any]=0.0_2				,		__A:     Optional[int]=6				,		__A:     Union[str, Any]=6				,		__A:     Union[str, Any]=3				,		__A:     Tuple=4				,		__A:     Optional[int]=None				,		__A:     Optional[Any]=1000				,		):

												'''simple docstring'''

												a__          =  parent
												a__          =  batch_size
												a__          =  num_channels
												a__          =  image_size
												a__          =  patch_size
												a__          =  is_training
												a__          =  use_input_mask
												a__          =  use_token_type_ids
												a__          =  use_labels
												a__          =  vocab_size
												a__          =  hidden_size
												a__          =  num_hidden_layers
												a__          =  num_attention_heads
												a__          =  intermediate_size
												a__          =  hidden_act
												a__          =  hidden_dropout_prob
												a__          =  attention_probs_dropout_prob
												a__          =  max_position_embeddings
												a__          =  type_vocab_size
												a__          =  type_sequence_label_size
												a__          =  initializer_range
												a__          =  coordinate_size
												a__          =  shape_size
												a__          =  num_labels
												a__          =  num_choices
												a__          =  scope
												a__          =  range_bbox

												# LayoutLMv3's sequence length equals the number of text tokens + number of patches + 1 (we add 1 for the CLS token)
												a__          =  text_seq_length
												a__          =  (image_size // patch_size) ** 2 + 1
												a__          =  self.text_seq_length + self.image_seq_length




							def  lowercase		(   self:     Optional[int]    ):

												'''simple docstring'''

												a__          =  ids_tensor([self.batch_size, self.text_seq_length]				,		self.vocab_size    )

												a__          =  ids_tensor([self.batch_size, self.text_seq_length, 4]				,		self.range_bbox    )
												a__          =  bbox.numpy()
												# Ensure that bbox is legal
												for i in range(bbox.shape[0]    ):
																	for j in range(bbox.shape[1]    ):
																						if bbox[i, j, 3] < bbox[i, j, 1]:
																											a__          =  bbox[i, j, 3]
																											a__          =  bbox[i, j, 1]
																											a__          =  tmp_coordinate
																						if bbox[i, j, 2] < bbox[i, j, 0]:
																											a__          =  bbox[i, j, 2]
																											a__          =  bbox[i, j, 0]
																											a__          =  tmp_coordinate
												a__          =  tf.constant(__A    )

												a__          =  floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]    )

												a__          =  None
												if self.use_input_mask:
																	a__          =  random_attention_mask([self.batch_size, self.text_seq_length]    )

												a__          =  None
												if self.use_token_type_ids:
																	a__          =  ids_tensor([self.batch_size, self.text_seq_length]				,		self.type_vocab_size    )

												a__          =  None
												a__          =  None
												if self.use_labels:
																	a__          =  ids_tensor([self.batch_size]				,		self.type_sequence_label_size    )
																	a__          =  ids_tensor([self.batch_size, self.text_seq_length]				,		self.num_labels    )

												a__          =  LayoutLMvaConfig(
												    vocab_size=self.vocab_size				,		hidden_size=self.hidden_size				,		num_hidden_layers=self.num_hidden_layers				,		num_attention_heads=self.num_attention_heads				,		intermediate_size=self.intermediate_size				,		hidden_act=self.hidden_act				,		hidden_dropout_prob=self.hidden_dropout_prob				,		attention_probs_dropout_prob=self.attention_probs_dropout_prob				,		max_position_embeddings=self.max_position_embeddings				,		type_vocab_size=self.type_vocab_size				,		initializer_range=self.initializer_range				,		coordinate_size=self.coordinate_size				,		shape_size=self.shape_size				,		input_size=self.image_size				,		patch_size=self.patch_size				,		)

												return config, input_ids, bbox, pixel_values, token_type_ids, input_mask, sequence_labels, token_labels




							def  lowercase		(   self:     List[str]				,		__A:     List[str]				,		__A:     List[str]				,		__A:     List[str]				,		__A:     int				,		__A:     Any				,		__A:     Any    ):

												'''simple docstring'''

												a__          =  TFLayoutLMvaModel(config=__A    )

												# text + image
												a__          =  model(__A				,		pixel_values=__A				,		training=__A    )
												a__          =  model(
												    __A				,		bbox=__A				,		pixel_values=__A				,		attention_mask=__A				,		token_type_ids=__A				,		training=__A				,		)
												a__          =  model(__A				,		bbox=__A				,		pixel_values=__A				,		training=__A    )

												self.parent.assertEqual(result.last_hidden_state.shape				,		(self.batch_size, self.seq_length, self.hidden_size)    )

												# text only
												a__          =  model(__A				,		training=__A    )
												self.parent.assertEqual(
												    result.last_hidden_state.shape				,		(self.batch_size, self.text_seq_length, self.hidden_size)    )

												# image only
												a__          =  model({'''pixel_values''': pixel_values}				,		training=__A    )
												self.parent.assertEqual(
												    result.last_hidden_state.shape				,		(self.batch_size, self.image_seq_length, self.hidden_size)    )




							def  lowercase		(   self:     Optional[int]				,		__A:     Any				,		__A:     str				,		__A:     List[str]				,		__A:     List[str]				,		__A:     List[str]				,		__A:     Optional[Any]				,		__A:     Any    ):

												'''simple docstring'''

												a__          =  self.num_labels
												a__          =  TFLayoutLMvaForSequenceClassification(config=__A    )
												a__          =  model(
												    __A				,		bbox=__A				,		pixel_values=__A				,		attention_mask=__A				,		token_type_ids=__A				,		labels=__A				,		training=__A				,		)
												self.parent.assertEqual(result.logits.shape				,		(self.batch_size, self.num_labels)    )




							def  lowercase		(   self:     str				,		__A:     List[str]				,		__A:     int				,		__A:     str				,		__A:     Any				,		__A:     str				,		__A:     str				,		__A:     str    ):

												'''simple docstring'''

												a__          =  self.num_labels
												a__          =  TFLayoutLMvaForTokenClassification(config=__A    )
												a__          =  model(
												    __A				,		bbox=__A				,		pixel_values=__A				,		attention_mask=__A				,		token_type_ids=__A				,		labels=__A				,		training=__A				,		)
												self.parent.assertEqual(result.logits.shape				,		(self.batch_size, self.text_seq_length, self.num_labels)    )




							def  lowercase		(   self:     Union[str, Any]				,		__A:     Any				,		__A:     List[Any]				,		__A:     Any				,		__A:     List[str]				,		__A:     Any				,		__A:     Optional[int]				,		__A:     Tuple    ):

												'''simple docstring'''

												a__          =  2
												a__          =  TFLayoutLMvaForQuestionAnswering(config=__A    )
												a__          =  model(
												    __A				,		bbox=__A				,		pixel_values=__A				,		attention_mask=__A				,		token_type_ids=__A				,		start_positions=__A				,		end_positions=__A				,		training=__A				,		)
												self.parent.assertEqual(result.start_logits.shape				,		(self.batch_size, self.seq_length)    )
												self.parent.assertEqual(result.end_logits.shape				,		(self.batch_size, self.seq_length)    )





							def  lowercase		(   self:     int    ):

												'''simple docstring'''

												a__          =  self.prepare_config_and_inputs()
												((a__)       ,(a__)       ,(a__)       ,(a__)       ,(a__)       ,(a__)       ,(a__)       ,(a__))          =  config_and_inputs
												a__          =  {
												    '''input_ids''': input_ids,
												    '''bbox''': bbox,
												    '''pixel_values''': pixel_values,
												    '''token_type_ids''': token_type_ids,
												    '''attention_mask''': input_mask,
												}
												return config, inputs_dict







@require_tf
class        _SCREAMING_SNAKE_CASE						(  __snake_case       , __snake_case       , unittest.TestCase							):





							"""simple docstring"""





							_SCREAMING_SNAKE_CASE    =(
							    (
							        TFLayoutLMvaModel,
							        TFLayoutLMvaForQuestionAnswering,
							        TFLayoutLMvaForSequenceClassification,
							        TFLayoutLMvaForTokenClassification,
							    )
							    if is_tf_available()
							    else ()
							)
							_SCREAMING_SNAKE_CASE    =(
							    {'document-question-answering': TFLayoutLMvaForQuestionAnswering, 'feature-extraction': TFLayoutLMvaModel}
							    if is_tf_available()
							    else {}
							)

							_SCREAMING_SNAKE_CASE    =False
							_SCREAMING_SNAKE_CASE    =False
							_SCREAMING_SNAKE_CASE    =False




							def  lowercase		(   self:     List[str]				,		__A:     Dict				,		__A:     Optional[Any]				,		__A:     str				,		__A:     Union[str, Any]				,		__A:     Optional[Any]    ):

												'''simple docstring'''

												return True




							def  lowercase		(   self:     Dict				,		__A:     Optional[Any]				,		__A:     Any				,		__A:     List[Any]=False    ):

												'''simple docstring'''

												a__          =  copy.deepcopy(__A    )

												if model_class in get_values(__A    ):
																	a__          =  {
																	    k: tf.tile(tf.expand_dims(__A				,		1    )				,		(1, self.model_tester.num_choices) + (1,) * (v.ndim - 1)    )
																	    if isinstance(__A				,		tf.Tensor    ) and v.ndim > 0
																	    else v
																	    for k, v in inputs_dict.items()
																	}

												if return_labels:
																	if model_class in get_values(__A    ):
																						a__          =  tf.ones(self.model_tester.batch_size				,		dtype=tf.intaa    )
																	elif model_class in get_values(__A    ):
																						a__          =  tf.zeros(self.model_tester.batch_size				,		dtype=tf.intaa    )
																						a__          =  tf.zeros(self.model_tester.batch_size				,		dtype=tf.intaa    )
																	elif model_class in get_values(__A    ):
																						a__          =  tf.zeros(self.model_tester.batch_size				,		dtype=tf.intaa    )
																	elif model_class in get_values(__A    ):
																						a__          =  tf.zeros(
																						    (self.model_tester.batch_size, self.model_tester.text_seq_length)				,		dtype=tf.intaa    )

												return inputs_dict




							def  lowercase		(   self:     List[str]    ):

												'''simple docstring'''

												a__          =  TFLayoutLMvaModelTester(self    )
												a__          =  ConfigTester(self				,		config_class=__A				,		hidden_size=37    )




							def  lowercase		(   self:     Dict    ):

												'''simple docstring'''

												self.config_tester.run_common_tests()




							def  lowercase		(   self:     Any    ):

												'''simple docstring'''

												a__       ,a__          =  self.model_tester.prepare_config_and_inputs_for_common()
												for model_class in self.all_model_classes:
																	a__          =  model_class(__A    )
																	if getattr(__A				,		'''hf_compute_loss'''				,		__A    ):
																						# The number of elements in the loss should be the same as the number of elements in the label
																						a__          =  self._prepare_for_class(inputs_dict.copy()				,		__A				,		return_labels=__A    )
																						a__          =  prepared_for_class[
																						    sorted(prepared_for_class.keys() - inputs_dict.keys()				,		reverse=__A    )[0]
																						]
																						a__          =  added_label.shape.as_list()[:1]

																						# Test that model correctly compute the loss with kwargs
																						a__          =  self._prepare_for_class(inputs_dict.copy()				,		__A				,		return_labels=__A    )
																						a__          =  prepared_for_class.pop('''input_ids'''    )

																						a__          =  model(__A				,		**__A    )[0]
																						self.assertTrue(loss.shape.as_list() == expected_loss_size or loss.shape.as_list() == [1]    )

																						# Test that model correctly compute the loss when we mask some positions
																						a__          =  self._prepare_for_class(inputs_dict.copy()				,		__A				,		return_labels=__A    )
																						a__          =  prepared_for_class.pop('''input_ids'''    )
																						if "labels" in prepared_for_class:
																											a__          =  prepared_for_class['''labels'''].numpy()
																											if len(labels.shape    ) > 1 and labels.shape[1] != 1:
																																a__          =  -100
																																a__          =  tf.convert_to_tensor(__A    )
																																a__          =  model(__A				,		**__A    )[0]
																																self.assertTrue(loss.shape.as_list() == expected_loss_size or loss.shape.as_list() == [1]    )
																																self.assertTrue(not np.any(np.isnan(loss.numpy()    )    )    )

                # Test that model correctly compute the loss with a dict
																						a__          =  self._prepare_for_class(inputs_dict.copy()				,		__A				,		return_labels=__A    )
																						a__          =  model(__A    )[0]
																						self.assertTrue(loss.shape.as_list() == expected_loss_size or loss.shape.as_list() == [1]    )

																						# Test that model correctly compute the loss with a tuple
																						a__          =  self._prepare_for_class(inputs_dict.copy()				,		__A				,		return_labels=__A    )

																						# Get keys that were added with the _prepare_for_class function
																						a__          =  prepared_for_class.keys() - inputs_dict.keys()
																						a__          =  inspect.signature(model.call    ).parameters
																						a__          =  list(signature.keys()    )

																						# Create a dictionary holding the location of the tensors in the tuple
																						a__          =  {0: '''input_ids'''}
																						for label_key in label_keys:
																											a__          =  signature_names.index(__A    )
																											a__          =  label_key
																						a__          =  sorted(tuple_index_mapping.items()    )
																						# Initialize a list with their default values, update the values and convert to a tuple
																						a__          =  []

																						for name in signature_names:
																											if name != "kwargs":
																																list_input.append(signature[name].default    )

																						for index, value in sorted_tuple_index_mapping:
																											a__          =  prepared_for_class[value]

																						a__          =  tuple(__A    )

																						# Send to model
																						a__          =  model(tuple_input[:-1]    )[0]

																						self.assertTrue(loss.shape.as_list() == expected_loss_size or loss.shape.as_list() == [1]    )




							def  lowercase		(   self:     Optional[int]    ):

												'''simple docstring'''

												(
												    (
												    a__
												)       ,(
												    a__
												)       ,(
												    a__
												)       ,(
												    a__
												)       ,(
												    a__
												)       ,(
												    a__
												)       ,(
												    a__
												)       ,(
												    a__
												)       ,
												)          =  self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_model(__A				,		__A				,		__A				,		__A				,		__A				,		__A    )




							def  lowercase		(   self:     Dict    ):

												'''simple docstring'''

												(
												    (
												    a__
												)       ,(
												    a__
												)       ,(
												    a__
												)       ,(
												    a__
												)       ,(
												    a__
												)       ,(
												    a__
												)       ,(
												    a__
												)       ,(
												    a__
												)       ,
												)          =  self.model_tester.prepare_config_and_inputs()
												for type in ["absolute", "relative_key", "relative_key_query"]:
																	a__          =  type
																	self.model_tester.create_and_check_model(__A				,		__A				,		__A				,		__A				,		__A				,		__A    )




							def  lowercase		(   self:     int    ):

												'''simple docstring'''

												(
												    (
												    a__
												)       ,(
												    a__
												)       ,(
												    a__
												)       ,(
												    a__
												)       ,(
												    a__
												)       ,(
												    a__
												)       ,(
												    a__
												)       ,(
												    a__
												)       ,
												)          =  self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_sequence_classification(
												    __A				,		__A				,		__A				,		__A				,		__A				,		__A				,		__A    )




							def  lowercase		(   self:     List[str]    ):

												'''simple docstring'''

												(
												    (
												    a__
												)       ,(
												    a__
												)       ,(
												    a__
												)       ,(
												    a__
												)       ,(
												    a__
												)       ,(
												    a__
												)       ,(
												    a__
												)       ,(
												    a__
												)       ,
												)          =  self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_token_classification(
												    __A				,		__A				,		__A				,		__A				,		__A				,		__A				,		__A    )




							def  lowercase		(   self:     Tuple    ):

												'''simple docstring'''

												(
												    (
												    a__
												)       ,(
												    a__
												)       ,(
												    a__
												)       ,(
												    a__
												)       ,(
												    a__
												)       ,(
												    a__
												)       ,(
												    a__
												)       ,(
												    a__
												)       ,
												)          =  self.model_tester.prepare_config_and_inputs()
												self.model_tester.create_and_check_for_question_answering(
												    __A				,		__A				,		__A				,		__A				,		__A				,		__A				,		__A    )





							@slow
							def  lowercase		(   self:     Union[str, Any]    ):

												'''simple docstring'''

												for model_name in TF_LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
																	a__          =  TFLayoutLMvaModel.from_pretrained(__A    )
																	self.assertIsNotNone(__A    )





def        SCREAMING_SNAKE_CASE       (      ):
					a__          =  Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png''')
					return image







@require_tf
class        _SCREAMING_SNAKE_CASE						(  unittest.TestCase							):





							"""simple docstring"""





							@cached_property
							def  lowercase		(   self:     Any    ):

												'''simple docstring'''

												return LayoutLMvaImageProcessor(apply_ocr=__A    ) if is_vision_available() else None





							@slow
							def  lowercase		(   self:     Union[str, Any]    ):

												'''simple docstring'''

												a__          =  TFLayoutLMvaModel.from_pretrained('''microsoft/layoutlmv3-base'''    )

												a__          =  self.default_image_processor
												a__          =  prepare_img()
												a__          =  image_processor(images=__A				,		return_tensors='''tf'''    ).pixel_values

												a__          =  tf.constant([[1, 2]]    )
												a__          =  tf.expand_dims(tf.constant([[1, 2, 3, 4], [5, 6, 7, 8]]    )				,		axis=0    )

												# forward pass
												a__          =  model(input_ids=__A				,		bbox=__A				,		pixel_values=__A				,		training=__A    )

												# verify the logits
												a__          =  (1, 199, 768)
												self.assertEqual(outputs.last_hidden_state.shape				,		__A    )

												a__          =  tf.constant(
												    [[-0.0_5_2_9, 0.3_6_1_8, 0.1_6_3_2], [-0.1_5_8_7, -0.1_6_6_7, -0.0_4_0_0], [-0.1_5_5_7, -0.1_6_7_1, -0.0_5_0_5]]    )

												self.assertTrue(np.allclose(outputs.last_hidden_state[0, :3, :3]				,		__A				,		atol=1e-4    )    )


