"""simple docstring"""






import argparse
import gc
import json
import os
import shutil
import warnings

import torch

from transformers import LlamaConfig, LlamaForCausalLM, LlamaTokenizer


try:
		from transformers import LlamaTokenizerFast
except ImportError as e:
		warnings.warn(e)
		warnings.warn(
		    '''The converted tokenizer will be the `slow` tokenizer. To use the fast, update your `tokenizers` library and re-run the tokenizer conversion'''
		)
		A 			=  None
A 			=  {
    '''7B''': 11_008,
    '''13B''': 13_824,
    '''30B''': 17_920,
    '''65B''': 22_016,
    '''70B''': 28_672,
}
A 			=  {
    '''7B''': 1,
    '''7Bf''': 1,
    '''13B''': 2,
    '''13Bf''': 2,
    '''30B''': 4,
    '''65B''': 8,
    '''70B''': 8,
    '''70Bf''': 8,
}

def     __A      (			a_						:Dict   ,       a_						:str=1   ,       a_						:List[str]=2_56)		->		Optional[int]:
				return multiple_of * ((int(ffn_dim_multiplier * int(8 * n / 3)) + multiple_of - 1) // multiple_of)

def     __A      (			a_						:List[str])		->		Optional[int]:
				with open(a_   ,       '''r''') as f:
								return json.load(a_)

def     __A      (			a_						:List[Any]   ,       a_						:List[Any])		->		str:
				with open(a_   ,       '''w''') as f:
								json.dump(a_   ,       a_)

def     __A      (			a_						:int   ,       a_						:Dict   ,       a_						:List[Any]   ,       a_						:Any=True)		->		Tuple:
				os.makedirs(a_   ,       exist_ok=a_)
				__a :						Any			       =  os.path.join(a_   ,       '''tmp''')
				os.makedirs(a_   ,       exist_ok=a_)

				__a :						Union[str, Any]			       =  read_json(os.path.join(a_   ,       '''params.json'''))
				__a :						List[str]			       =  NUM_SHARDS[model_size]
				__a :						Union[str, Any]			       =  params['''n_layers''']
				__a :						List[Any]			       =  params['''n_heads''']
				__a :						Any			       =  n_heads // num_shards
				__a :						Tuple			       =  params['''dim''']
				__a :						List[Any]			       =  dim // n_heads
				__a :						List[Any]			       =  1_00_00.0
				__a :						List[Any]			       =  1.0 / (base ** (torch.arange(0   ,       a_   ,       2).float() / dims_per_head))

				if "n_kv_heads" in params:
								__a :						int			       =  params['''n_kv_heads''']  # for GQA / MQA
								__a :						Optional[Any]			       =  n_heads_per_shard // num_key_value_heads
								__a :						Any			       =  dim // num_key_value_heads
				else:  # compatibility with other checkpoints
								__a :						List[Any]			       =  n_heads
								__a :						int			       =  n_heads_per_shard
								__a :						Union[str, Any]			       =  dim

				# permute for sliced rotary
				def permute(a_						:Any   ,       a_						:Optional[int]=n_heads   ,       a_						:Any=dim   ,       a_						:int=dim):
								return w.view(a_   ,       dima // n_heads // 2   ,       2   ,       a_).transpose(1   ,       2).reshape(a_   ,       a_)

				print(F"""Fetching all parameters from the checkpoint at {input_base_path}.""")
				# Load weights
				if model_size == "7B":
								# Not sharded
								# (The sharded implementation would also work, but this is simpler.)
								__a :						Any			       =  torch.load(os.path.join(a_   ,       '''consolidated.00.pth''')   ,       map_location='''cpu''')
				else:
								# Sharded
								__a :						Dict			       =  [
								    torch.load(os.path.join(a_   ,       F"""consolidated.{i:02d}.pth""")   ,       map_location='''cpu''')
								    for i in range(a_)
								]
				__a :						Tuple			       =  0
				__a :						Any			       =  {'''weight_map''': {}}
				for layer_i in range(a_):
								__a :						Optional[int]			       =  F"""pytorch_model-{layer_i + 1}-of-{n_layers + 1}.bin"""
								if model_size == "7B":
												# Unsharded
												__a :						Tuple			       =  {
												    F"""model.layers.{layer_i}.self_attn.q_proj.weight""": permute(
												        loaded[F"""layers.{layer_i}.attention.wq.weight"""]),
												    F"""model.layers.{layer_i}.self_attn.k_proj.weight""": permute(
												        loaded[F"""layers.{layer_i}.attention.wk.weight"""]),
												    F"""model.layers.{layer_i}.self_attn.v_proj.weight""": loaded[F"""layers.{layer_i}.attention.wv.weight"""],
												    F"""model.layers.{layer_i}.self_attn.o_proj.weight""": loaded[F"""layers.{layer_i}.attention.wo.weight"""],
												    F"""model.layers.{layer_i}.mlp.gate_proj.weight""": loaded[F"""layers.{layer_i}.feed_forward.w1.weight"""],
												    F"""model.layers.{layer_i}.mlp.down_proj.weight""": loaded[F"""layers.{layer_i}.feed_forward.w2.weight"""],
												    F"""model.layers.{layer_i}.mlp.up_proj.weight""": loaded[F"""layers.{layer_i}.feed_forward.w3.weight"""],
												    F"""model.layers.{layer_i}.input_layernorm.weight""": loaded[F"""layers.{layer_i}.attention_norm.weight"""],
												    F"""model.layers.{layer_i}.post_attention_layernorm.weight""": loaded[F"""layers.{layer_i}.ffn_norm.weight"""],
												}
								else:
												# Sharded
												# Note that attention.w{q,k,v,o}, feed_fordward.w[1,2,3], attention_norm.weight and ffn_norm.weight share
												# the same storage object, saving attention_norm and ffn_norm will save other weights too, which is
												# redundant as other weights will be stitched from multiple shards. To avoid that, they are cloned.

												__a :						int			       =  {
												    F"""model.layers.{layer_i}.input_layernorm.weight""": loaded[0][
												        F"""layers.{layer_i}.attention_norm.weight"""
												    ].clone(),
												    F"""model.layers.{layer_i}.post_attention_layernorm.weight""": loaded[0][
												        F"""layers.{layer_i}.ffn_norm.weight"""
												    ].clone(),
												}
												__a :						Union[str, Any]			       =  permute(
												    torch.cat(
												        [
												            loaded[i][F"""layers.{layer_i}.attention.wq.weight"""].view(a_   ,       a_   ,       a_)
												            for i in range(a_)
												        ]   ,       dim=0   ,       ).reshape(a_   ,       a_))
												__a :						Optional[Any]			       =  permute(
												    torch.cat(
												        [
												            loaded[i][F"""layers.{layer_i}.attention.wk.weight"""].view(
												                a_   ,       a_   ,       a_)
												            for i in range(a_)
												        ]   ,       dim=0   ,       ).reshape(a_   ,       a_)   ,       a_   ,       a_   ,       a_   ,       )
												__a :						Dict			       =  torch.cat(
												    [
												        loaded[i][F"""layers.{layer_i}.attention.wv.weight"""].view(
												            a_   ,       a_   ,       a_)
												        for i in range(a_)
												    ]   ,       dim=0   ,       ).reshape(a_   ,       a_)

												__a :						Dict			       =  torch.cat(
												    [loaded[i][F"""layers.{layer_i}.attention.wo.weight"""] for i in range(a_)]   ,       dim=1)
												__a :						List[Any]			       =  torch.cat(
												    [loaded[i][F"""layers.{layer_i}.feed_forward.w1.weight"""] for i in range(a_)]   ,       dim=0)
												__a :						List[Any]			       =  torch.cat(
												    [loaded[i][F"""layers.{layer_i}.feed_forward.w2.weight"""] for i in range(a_)]   ,       dim=1)
												__a :						List[str]			       =  torch.cat(
												    [loaded[i][F"""layers.{layer_i}.feed_forward.w3.weight"""] for i in range(a_)]   ,       dim=0)

								__a :						Union[str, Any]			       =  inv_freq
								for k, v in state_dict.items():
												__a :						List[Any]			       =  filename
												param_count += v.numel()
								torch.save(a_   ,       os.path.join(a_   ,       a_))

				__a :						List[Any]			       =  F"""pytorch_model-{n_layers + 1}-of-{n_layers + 1}.bin"""
				if model_size == "7B":
								# Unsharded
								__a :						Optional[int]			       =  {
								    '''model.embed_tokens.weight''': loaded['''tok_embeddings.weight'''],
								    '''model.norm.weight''': loaded['''norm.weight'''],
								    '''lm_head.weight''': loaded['''output.weight'''],
								}
				else:
								__a :						List[Any]			       =  {
								    '''model.norm.weight''': loaded[0]['''norm.weight'''],
								    '''model.embed_tokens.weight''': torch.cat(
								        [loaded[i]['''tok_embeddings.weight'''] for i in range(a_)]   ,       dim=1),
								    '''lm_head.weight''': torch.cat([loaded[i]['''output.weight'''] for i in range(a_)]   ,       dim=0),
								}

				for k, v in state_dict.items():
								__a :						Any			       =  filename
								param_count += v.numel()
				torch.save(a_   ,       os.path.join(a_   ,       a_))

				# Write configs
				__a :						Optional[int]			       =  {'''total_size''': param_count * 2}
				write_json(a_   ,       os.path.join(a_   ,       '''pytorch_model.bin.index.json'''))
				__a :						Union[str, Any]			       =  params['''ffn_dim_multiplier'''] if '''ffn_dim_multiplier''' in params else 1
				__a :						Any			       =  params['''multiple_of'''] if '''multiple_of''' in params else 2_56
				__a :						Optional[int]			       =  LlamaConfig(
				    hidden_size=a_   ,       intermediate_size=compute_intermediate_size(a_   ,       a_   ,       a_)   ,       num_attention_heads=params['''n_heads''']   ,       num_hidden_layers=params['''n_layers''']   ,       rms_norm_eps=params['''norm_eps''']   ,       num_key_value_heads=a_   ,       )
				config.save_pretrained(a_)

				# Make space so we can load the model properly now.
				del state_dict
				del loaded
				gc.collect()

				print('''Loading the checkpoint in a Llama model.''')
				__a :						str			       =  LlamaForCausalLM.from_pretrained(a_   ,       torch_dtype=torch.floataa   ,       low_cpu_mem_usage=a_)
				# Avoid saving this as part of the config.
				del model.config._name_or_path

				print('''Saving in the Transformers format.''')
				model.save_pretrained(a_   ,       safe_serialization=a_)
				shutil.rmtree(a_)

def     __A      (			a_						:Any   ,       a_						:str)		->		int:
				# Initialize the tokenizer based on the `spm` model
				__a :						Optional[int]			       =  LlamaTokenizer if LlamaTokenizerFast is None else LlamaTokenizerFast
				print(F"""Saving a {tokenizer_class.__name__} to {tokenizer_path}.""")
				__a :						str			       =  tokenizer_class(a_)
				tokenizer.save_pretrained(a_)

def     __A      (			)		->		Tuple:
				__a :						Optional[int]			       =  argparse.ArgumentParser()
				parser.add_argument(
				    '''--input_dir'''   ,       help='''Location of LLaMA weights, which contains tokenizer.model and model folders'''   ,       )
				parser.add_argument(
				    '''--model_size'''   ,       choices=['''7B''', '''7Bf''', '''13B''', '''13Bf''', '''30B''', '''65B''', '''70B''', '''70Bf''', '''tokenizer_only''']   ,       )
				parser.add_argument(
				    '''--output_dir'''   ,       help='''Location to write HF model and tokenizer'''   ,       )
				parser.add_argument('''--safe_serialization'''   ,       type=a_   ,       help='''Whether or not to save using `safetensors`.''')
				__a :						Optional[Any]			       =  parser.parse_args()
				if args.model_size != "tokenizer_only":
								write_model(
								    model_path=args.output_dir   ,       input_base_path=os.path.join(args.input_dir   ,       args.model_size)   ,       model_size=args.model_size   ,       safe_serialization=args.safe_serialization   ,       )
				__a :						Optional[Any]			       =  os.path.join(args.input_dir   ,       '''tokenizer.model''')
				write_tokenizer(args.output_dir   ,       a_)


if __name__ == "__main__":
		main()






"""simple docstring"""






import copy
import unittest

from transformers.models.auto import get_values
from transformers.testing_utils import require_torch, slow, torch_device
from transformers.utils import cached_property, is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, floats_tensor, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import (
		    MODEL_FOR_MULTIPLE_CHOICE_MAPPING,
		    MODEL_FOR_QUESTION_ANSWERING_MAPPING,
		    MODEL_FOR_SEQUENCE_CLASSIFICATION_MAPPING,
		    MODEL_FOR_TOKEN_CLASSIFICATION_MAPPING,
		    LayoutLMvaConfig,
		    LayoutLMvaForQuestionAnswering,
		    LayoutLMvaForSequenceClassification,
		    LayoutLMvaForTokenClassification,
		    LayoutLMvaModel,
		)
		from transformers.models.layoutlmva.modeling_layoutlmva import LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST

if is_vision_available():
		from PIL import Image

		from transformers import LayoutLMvaImageProcessor




class 				__lowercase			:







					'''simple docstring'''




					def __init__(						self	,			_UpperCAmelCase	,			_UpperCAmelCase=2	,			_UpperCAmelCase=3	,			_UpperCAmelCase=4	,			_UpperCAmelCase=2	,			_UpperCAmelCase=7	,			_UpperCAmelCase=True	,			_UpperCAmelCase=True	,			_UpperCAmelCase=True	,			_UpperCAmelCase=True	,			_UpperCAmelCase=99	,			_UpperCAmelCase=36	,			_UpperCAmelCase=3	,			_UpperCAmelCase=4	,			_UpperCAmelCase=37	,			_UpperCAmelCase="gelu"	,			_UpperCAmelCase=0.1	,			_UpperCAmelCase=0.1	,			_UpperCAmelCase=512	,			_UpperCAmelCase=16	,			_UpperCAmelCase=2	,			_UpperCAmelCase=0.0_2	,			_UpperCAmelCase=6	,			_UpperCAmelCase=6	,			_UpperCAmelCase=3	,			_UpperCAmelCase=4	,			_UpperCAmelCase=None	,			_UpperCAmelCase=1000	,			):
									__a :						Dict			       =  parent
									__a :						Optional[int]			       =  batch_size
									__a :						Optional[int]			       =  num_channels
									__a :						List[Any]			       =  image_size
									__a :						int			       =  patch_size
									__a :						Tuple			       =  text_seq_length
									__a :						Dict			       =  is_training
									__a :						str			       =  use_input_mask
									__a :						Optional[int]			       =  use_token_type_ids
									__a :						List[Any]			       =  use_labels
									__a :						Tuple			       =  vocab_size
									__a :						str			       =  hidden_size
									__a :						Any			       =  num_hidden_layers
									__a :						List[str]			       =  num_attention_heads
									__a :						str			       =  intermediate_size
									__a :						int			       =  hidden_act
									__a :						List[Any]			       =  hidden_dropout_prob
									__a :						Tuple			       =  attention_probs_dropout_prob
									__a :						Tuple			       =  max_position_embeddings
									__a :						List[Any]			       =  type_vocab_size
									__a :						int			       =  type_sequence_label_size
									__a :						str			       =  initializer_range
									__a :						Dict			       =  coordinate_size
									__a :						int			       =  shape_size
									__a :						int			       =  num_labels
									__a :						Optional[int]			       =  num_choices
									__a :						Any			       =  scope
									__a :						Any			       =  range_bbox

									# LayoutLMv3's sequence length equals the number of text tokens + number of patches + 1 (we add 1 for the CLS token)
									__a :						Optional[int]			       =  text_seq_length
									__a :						str			       =  (image_size // patch_size) ** 2 + 1
									__a :						Tuple			       =  self.text_seq_length + self.image_seq_length







					def  _lowerCamelCase		(						self     ):
									__a :						List[Any]			       =  ids_tensor([self.batch_size, self.text_seq_length]	,			self.vocab_size     )

									__a :						Union[str, Any]			       =  ids_tensor([self.batch_size, self.text_seq_length, 4]	,			self.range_bbox     )
									# Ensure that bbox is legal
									for i in range(bbox.shape[0]     ):
													for j in range(bbox.shape[1]     ):
																	if bbox[i, j, 3] < bbox[i, j, 1]:
																					__a :						Optional[Any]			       =  bbox[i, j, 3]
																					__a :						Union[str, Any]			       =  bbox[i, j, 1]
																					__a :						Tuple			       =  t
																	if bbox[i, j, 2] < bbox[i, j, 0]:
																					__a :						Optional[int]			       =  bbox[i, j, 2]
																					__a :						Optional[int]			       =  bbox[i, j, 0]
																					__a :						str			       =  t

									__a :						str			       =  floats_tensor([self.batch_size, self.num_channels, self.image_size, self.image_size]     )

									__a :						Union[str, Any]			       =  None
									if self.use_input_mask:
													__a :						int			       =  random_attention_mask([self.batch_size, self.text_seq_length]     )

									__a :						int			       =  None
									if self.use_token_type_ids:
													__a :						Union[str, Any]			       =  ids_tensor([self.batch_size, self.text_seq_length]	,			self.type_vocab_size     )

									__a :						Optional[int]			       =  None
									__a :						Dict			       =  None
									if self.use_labels:
													__a :						Optional[Any]			       =  ids_tensor([self.batch_size]	,			self.type_sequence_label_size     )
													__a :						Optional[int]			       =  ids_tensor([self.batch_size, self.text_seq_length]	,			self.num_labels     )

									__a :						int			       =  LayoutLMvaConfig(
									    vocab_size=self.vocab_size	,			hidden_size=self.hidden_size	,			num_hidden_layers=self.num_hidden_layers	,			num_attention_heads=self.num_attention_heads	,			intermediate_size=self.intermediate_size	,			hidden_act=self.hidden_act	,			hidden_dropout_prob=self.hidden_dropout_prob	,			attention_probs_dropout_prob=self.attention_probs_dropout_prob	,			max_position_embeddings=self.max_position_embeddings	,			type_vocab_size=self.type_vocab_size	,			initializer_range=self.initializer_range	,			coordinate_size=self.coordinate_size	,			shape_size=self.shape_size	,			input_size=self.image_size	,			patch_size=self.patch_size	,			)

									return config, input_ids, bbox, pixel_values, token_type_ids, input_mask, sequence_labels, token_labels







					def  _lowerCamelCase		(						self	,			_UpperCAmelCase	,			_UpperCAmelCase	,			_UpperCAmelCase	,			_UpperCAmelCase	,			_UpperCAmelCase	,			_UpperCAmelCase	,			_UpperCAmelCase	,			_UpperCAmelCase     ):
									__a :						Union[str, Any]			       =  LayoutLMvaModel(config=_UpperCAmelCase     )
									model.to(_UpperCAmelCase     )
									model.eval()

									# text + image
									__a :						Union[str, Any]			       =  model(_UpperCAmelCase	,			pixel_values=_UpperCAmelCase     )
									__a :						List[Any]			       =  model(
									    _UpperCAmelCase	,			bbox=_UpperCAmelCase	,			pixel_values=_UpperCAmelCase	,			attention_mask=_UpperCAmelCase	,			token_type_ids=_UpperCAmelCase     )
									__a :						Optional[int]			       =  model(_UpperCAmelCase	,			bbox=_UpperCAmelCase	,			pixel_values=_UpperCAmelCase	,			token_type_ids=_UpperCAmelCase     )
									__a :						List[str]			       =  model(_UpperCAmelCase	,			bbox=_UpperCAmelCase	,			pixel_values=_UpperCAmelCase     )

									self.parent.assertEqual(result.last_hidden_state.shape	,			(self.batch_size, self.seq_length, self.hidden_size)     )

									# text only
									__a :						int			       =  model(_UpperCAmelCase     )
									self.parent.assertEqual(
									    result.last_hidden_state.shape	,			(self.batch_size, self.text_seq_length, self.hidden_size)     )

									# image only
									__a :						Tuple			       =  model(pixel_values=_UpperCAmelCase     )
									self.parent.assertEqual(
									    result.last_hidden_state.shape	,			(self.batch_size, self.image_seq_length, self.hidden_size)     )







					def  _lowerCamelCase		(						self	,			_UpperCAmelCase	,			_UpperCAmelCase	,			_UpperCAmelCase	,			_UpperCAmelCase	,			_UpperCAmelCase	,			_UpperCAmelCase	,			_UpperCAmelCase	,			_UpperCAmelCase     ):
									__a :						Optional[Any]			       =  self.num_labels
									__a :						Optional[Any]			       =  LayoutLMvaForSequenceClassification(_UpperCAmelCase     )
									model.to(_UpperCAmelCase     )
									model.eval()
									__a :						Union[str, Any]			       =  model(
									    _UpperCAmelCase	,			bbox=_UpperCAmelCase	,			pixel_values=_UpperCAmelCase	,			attention_mask=_UpperCAmelCase	,			token_type_ids=_UpperCAmelCase	,			labels=_UpperCAmelCase	,			)
									self.parent.assertEqual(result.logits.shape	,			(self.batch_size, self.num_labels)     )







					def  _lowerCamelCase		(						self	,			_UpperCAmelCase	,			_UpperCAmelCase	,			_UpperCAmelCase	,			_UpperCAmelCase	,			_UpperCAmelCase	,			_UpperCAmelCase	,			_UpperCAmelCase	,			_UpperCAmelCase     ):
									__a :						List[str]			       =  self.num_labels
									__a :						int			       =  LayoutLMvaForTokenClassification(config=_UpperCAmelCase     )
									model.to(_UpperCAmelCase     )
									model.eval()
									__a :						List[str]			       =  model(
									    _UpperCAmelCase	,			bbox=_UpperCAmelCase	,			pixel_values=_UpperCAmelCase	,			attention_mask=_UpperCAmelCase	,			token_type_ids=_UpperCAmelCase	,			labels=_UpperCAmelCase	,			)
									self.parent.assertEqual(result.logits.shape	,			(self.batch_size, self.text_seq_length, self.num_labels)     )







					def  _lowerCamelCase		(						self	,			_UpperCAmelCase	,			_UpperCAmelCase	,			_UpperCAmelCase	,			_UpperCAmelCase	,			_UpperCAmelCase	,			_UpperCAmelCase	,			_UpperCAmelCase	,			_UpperCAmelCase     ):
									__a :						List[Any]			       =  LayoutLMvaForQuestionAnswering(config=_UpperCAmelCase     )
									model.to(_UpperCAmelCase     )
									model.eval()
									__a :						List[str]			       =  model(
									    _UpperCAmelCase	,			bbox=_UpperCAmelCase	,			pixel_values=_UpperCAmelCase	,			attention_mask=_UpperCAmelCase	,			token_type_ids=_UpperCAmelCase	,			start_positions=_UpperCAmelCase	,			end_positions=_UpperCAmelCase	,			)
									self.parent.assertEqual(result.start_logits.shape	,			(self.batch_size, self.seq_length)     )
									self.parent.assertEqual(result.end_logits.shape	,			(self.batch_size, self.seq_length)     )







					def  _lowerCamelCase		(						self     ):
									__a :						Optional[int]			       =  self.prepare_config_and_inputs()
									(
									    (
									    __a
									)       ,						(
									    __a
									)       ,						(
									    __a
									)       ,						(
									    __a
									)       ,						(
									    __a
									)       ,						(
									    __a
									)       ,						(
									    __a
									)       ,						(
									    __a
									)       ,						
									) :						List[Any]			       =  config_and_inputs
									__a :						Optional[int]			       =  {
									    '''input_ids''': input_ids,
									    '''bbox''': bbox,
									    '''pixel_values''': pixel_values,
									    '''token_type_ids''': token_type_ids,
									    '''attention_mask''': input_mask,
									}
									return config, inputs_dict






@require_torch
class 				__lowercase			(			_UpperCamelCase   ,      _UpperCamelCase   ,      unittest.TestCase							):







					'''simple docstring'''




					__lowerCAmelCase  =  False
					__lowerCAmelCase  =  False
					__lowerCAmelCase  =  False

					__lowerCAmelCase  =  (
					    (
					        LayoutLMvaModel,
					        LayoutLMvaForSequenceClassification,
					        LayoutLMvaForTokenClassification,
					        LayoutLMvaForQuestionAnswering,
					    )
					    if is_torch_available()
					    else ()
					)
					__lowerCAmelCase  =  (
					    {'''document-question-answering''': LayoutLMvaForQuestionAnswering, '''feature-extraction''': LayoutLMvaModel}
					    if is_torch_available()
					    else {}
					)







					def  _lowerCamelCase		(						self	,			_UpperCAmelCase	,			_UpperCAmelCase	,			_UpperCAmelCase	,			_UpperCAmelCase	,			_UpperCAmelCase     ):
									# `DocumentQuestionAnsweringPipeline` is expected to work with this model, but it combines the text and visual
									# embedding along the sequence dimension (dim 1), which causes an error during post-processing as `p_mask` has
									# the sequence dimension of the text embedding only.
									# (see the line `embedding_output = torch.cat([embedding_output, visual_embeddings], dim=1)`)
									return True







					def  _lowerCamelCase		(						self     ):
									__a :						str			       =  LayoutLMvaModelTester(self     )
									__a :						int			       =  ConfigTester(self	,			config_class=_UpperCAmelCase	,			hidden_size=37     )







					def  _lowerCamelCase		(						self	,			_UpperCAmelCase	,			_UpperCAmelCase	,			_UpperCAmelCase=False     ):
									__a :						int			       =  copy.deepcopy(_UpperCAmelCase     )
									if model_class in get_values(_UpperCAmelCase     ):
													__a :						List[str]			       =  {
													    k: v.unsqueeze(1     ).expand(-1	,			self.model_tester.num_choices	,			-1     ).contiguous()
													    if isinstance(_UpperCAmelCase	,			torch.Tensor     ) and v.ndim > 1
													    else v
													    for k, v in inputs_dict.items()
													}
									if return_labels:
													if model_class in get_values(_UpperCAmelCase     ):
																	__a :						Dict			       =  torch.ones(self.model_tester.batch_size	,			dtype=torch.long	,			device=_UpperCAmelCase     )
													elif model_class in get_values(_UpperCAmelCase     ):
																	__a :						int			       =  torch.zeros(
																	    self.model_tester.batch_size	,			dtype=torch.long	,			device=_UpperCAmelCase     )
																	__a :						Any			       =  torch.zeros(
																	    self.model_tester.batch_size	,			dtype=torch.long	,			device=_UpperCAmelCase     )
													elif model_class in [
													    *get_values(_UpperCAmelCase     ),
													]:
																	__a :						Any			       =  torch.zeros(
																	    self.model_tester.batch_size	,			dtype=torch.long	,			device=_UpperCAmelCase     )
													elif model_class in [
													    *get_values(_UpperCAmelCase     ),
													]:
																	__a :						int			       =  torch.zeros(
																	    (self.model_tester.batch_size, self.model_tester.text_seq_length)	,			dtype=torch.long	,			device=_UpperCAmelCase	,			)

									return inputs_dict







					def  _lowerCamelCase		(						self     ):
									self.config_tester.run_common_tests()







					def  _lowerCamelCase		(						self     ):
									__a :						List[str]			       =  self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_model(*_UpperCAmelCase     )







					def  _lowerCamelCase		(						self     ):
									__a :						Optional[int]			       =  self.model_tester.prepare_config_and_inputs()
									for type in ["absolute", "relative_key", "relative_key_query"]:
													__a :						Optional[Any]			       =  type
													self.model_tester.create_and_check_model(*_UpperCAmelCase     )







					def  _lowerCamelCase		(						self     ):
									__a :						Tuple			       =  self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_sequence_classification(*_UpperCAmelCase     )







					def  _lowerCamelCase		(						self     ):
									__a :						Tuple			       =  self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_token_classification(*_UpperCAmelCase     )







					def  _lowerCamelCase		(						self     ):
									__a :						int			       =  self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_for_question_answering(*_UpperCAmelCase     )







					@slow
					def  _lowerCamelCase		(						self     ):
									for model_name in LAYOUTLMV3_PRETRAINED_MODEL_ARCHIVE_LIST[:1]:
													__a :						Tuple			       =  LayoutLMvaModel.from_pretrained(_UpperCAmelCase     )
													self.assertIsNotNone(_UpperCAmelCase     )






def     __A      (			)		->		Dict:
				__a :						Tuple			       =  Image.open('''./tests/fixtures/tests_samples/COCO/000000039769.png''')
				return image






@require_torch
class 				__lowercase			(			unittest.TestCase							):







					'''simple docstring'''




					@cached_property
					def  _lowerCamelCase		(						self     ):
									return LayoutLMvaImageProcessor(apply_ocr=_UpperCAmelCase     ) if is_vision_available() else None







					@slow
					def  _lowerCamelCase		(						self     ):
									__a :						List[Any]			       =  LayoutLMvaModel.from_pretrained('''microsoft/layoutlmv3-base'''     ).to(_UpperCAmelCase     )

									__a :						Dict			       =  self.default_image_processor
									__a :						Tuple			       =  prepare_img()
									__a :						Optional[Any]			       =  image_processor(images=_UpperCAmelCase	,			return_tensors='''pt'''     ).pixel_values.to(_UpperCAmelCase     )

									__a :						Tuple			       =  torch.tensor([[1, 2]]     )
									__a :						Dict			       =  torch.tensor([[1, 2, 3, 4], [5, 6, 7, 8]]     ).unsqueeze(0     )

									# forward pass
									__a :						Tuple			       =  model(
									    input_ids=input_ids.to(_UpperCAmelCase     )	,			bbox=bbox.to(_UpperCAmelCase     )	,			pixel_values=pixel_values.to(_UpperCAmelCase     )	,			)

									# verify the logits
									__a :						Any			       =  torch.Size((1, 199, 768)     )
									self.assertEqual(outputs.last_hidden_state.shape	,			_UpperCAmelCase     )

									__a :						Union[str, Any]			       =  torch.tensor(
									    [[-0.0_5_2_9, 0.3_6_1_8, 0.1_6_3_2], [-0.1_5_8_7, -0.1_6_6_7, -0.0_4_0_0], [-0.1_5_5_7, -0.1_6_7_1, -0.0_5_0_5]]     ).to(_UpperCAmelCase     )

									self.assertTrue(torch.allclose(outputs.last_hidden_state[0, :3, :3]	,			_UpperCAmelCase	,			atol=1e-4     )     )