
"""simple docstring"""




import inspect
import unittest

import requests

from transformers import MgpstrConfig
from transformers.testing_utils import require_torch, require_vision, slow, torch_device
from transformers.utils import is_torch_available, is_vision_available

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, _config_zero_init, floats_tensor
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
	import torch
	from torch import nn

	from transformers import MgpstrForSceneTextRecognition


if is_vision_available():
	from PIL import Image

	from transformers import MgpstrProcessor




class      UpperCamelCase	:

				"""simple docstring"""



				def __init__(				self		,	lowercase__		,	lowercase__=False		,	lowercase__=13		,	lowercase__=(32, 128)		,	lowercase__=4		,	lowercase__=3		,	lowercase__=27		,	lowercase__=38		,	lowercase__=99		,	lowercase__=99		,	lowercase__=32		,	lowercase__=5		,	lowercase__=4		,	lowercase__=4.0		,	lowercase__=257		,	lowercase__=None		,	)    ->    List[str]:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE				=	parent
									SCREAMING_SNAKE_CASE				=	is_training
									SCREAMING_SNAKE_CASE				=	batch_size
									SCREAMING_SNAKE_CASE				=	image_size
									SCREAMING_SNAKE_CASE				=	patch_size
									SCREAMING_SNAKE_CASE				=	num_channels
									SCREAMING_SNAKE_CASE				=	max_token_length
									SCREAMING_SNAKE_CASE				=	num_character_labels
									SCREAMING_SNAKE_CASE				=	num_bpe_labels
									SCREAMING_SNAKE_CASE				=	num_wordpiece_labels
									SCREAMING_SNAKE_CASE				=	hidden_size
									SCREAMING_SNAKE_CASE				=	num_hidden_layers
									SCREAMING_SNAKE_CASE				=	num_attention_heads
									SCREAMING_SNAKE_CASE				=	mlp_ratio
									SCREAMING_SNAKE_CASE				=	patch_embeds_hidden_size
									SCREAMING_SNAKE_CASE				=	output_hidden_states


				def 							A					(				self			)    ->    str:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE				=	floats_tensor([self.batch_size, self.num_channels, self.image_size[0], self.image_size[1]]			)
									SCREAMING_SNAKE_CASE				=	self.get_config()
									return config, pixel_values


				def 							A					(				self			)    ->    Tuple:
									"""simple docstring"""







									return MgpstrConfig(
									    image_size=self.image_size		,	patch_size=self.patch_size		,	num_channels=self.num_channels		,	max_token_length=self.max_token_length		,	num_character_labels=self.num_character_labels		,	num_bpe_labels=self.num_bpe_labels		,	num_wordpiece_labels=self.num_wordpiece_labels		,	hidden_size=self.hidden_size		,	num_hidden_layers=self.num_hidden_layers		,	num_attention_heads=self.num_attention_heads		,	mlp_ratio=self.mlp_ratio		,	output_hidden_states=self.output_hidden_states		,	)


				def 							A					(				self		,	lowercase__		,	lowercase__			)    ->    Optional[Any]:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE				=	MgpstrForSceneTextRecognition(lowercase__			)
									model.to(lowercase__			)
									model.eval()
									with torch.no_grad():
														SCREAMING_SNAKE_CASE				=	model(lowercase__			)
									self.parent.assertEqual(
									    generated_ids[0][0].shape		,	(self.batch_size, self.max_token_length, self.num_character_labels)			)




				def 							A					(				self			)    ->    Dict:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE				=	self.prepare_config_and_inputs()
									SCREAMING_SNAKE_CASE	,    SCREAMING_SNAKE_CASE				=	config_and_inputs
									SCREAMING_SNAKE_CASE				=	{'pixel_values': pixel_values}
									return config, inputs_dict






@require_torch
class      UpperCamelCase	(       __magic_name__  ,      __magic_name__  ,      unittest.TestCase      ):

				"""simple docstring"""



				UpperCAmelCase_				:      Tuple						  =				(MgpstrForSceneTextRecognition,) if is_torch_available() else ()
				UpperCAmelCase_				:      Optional[Any]						  =				{"feature-extraction": MgpstrForSceneTextRecognition} if is_torch_available() else {}
				UpperCAmelCase_				:      List[Any]						  =				False

				UpperCAmelCase_				:      Any						  =				False
				UpperCAmelCase_				:      Dict						  =				False
				UpperCAmelCase_				:      Union[str, Any]						  =				False
				UpperCAmelCase_				:      str						  =				False


				def 							A					(				self			)    ->    Dict:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE				=	MgpstrModelTester(self			)
									SCREAMING_SNAKE_CASE				=	ConfigTester(self		,	config_class=lowercase__		,	has_text_modality=lowercase__			)


				def 							A					(				self			)    ->    Optional[Any]:
									"""simple docstring"""







									self.config_tester.run_common_tests()


				def 							A					(				self			)    ->    int:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE				=	self.model_tester.prepare_config_and_inputs()
									self.model_tester.create_and_check_model(*lowercase__			)


				@unittest.skip(reason='MgpstrModel does not use inputs_embeds'			)
				def 							A					(				self			)    ->    Optional[int]:
									"""simple docstring"""







									pass


				def 							A					(				self			)    ->    Tuple:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE	,    SCREAMING_SNAKE_CASE				=	self.model_tester.prepare_config_and_inputs_for_common()

									for model_class in self.all_model_classes:
														SCREAMING_SNAKE_CASE				=	model_class(lowercase__			)
														self.assertIsInstance(model.get_input_embeddings()		,	(nn.Module)			)
														SCREAMING_SNAKE_CASE				=	model.get_output_embeddings()
														self.assertTrue(x is None or isinstance(lowercase__		,	nn.Linear			)			)


				def 							A					(				self			)    ->    str:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE	,    SCREAMING_SNAKE_CASE				=	self.model_tester.prepare_config_and_inputs_for_common()

									for model_class in self.all_model_classes:
														SCREAMING_SNAKE_CASE				=	model_class(lowercase__			)
														SCREAMING_SNAKE_CASE				=	inspect.signature(model.forward			)
														# signature.parameters is an OrderedDict => so arg_names order is deterministic
														SCREAMING_SNAKE_CASE				=	[*signature.parameters.keys()]

														SCREAMING_SNAKE_CASE				=	['pixel_values']
														self.assertListEqual(arg_names[:1]		,	lowercase__			)


				@unittest.skip(reason='MgpstrModel does not support feedforward chunking'			)
				def 							A					(				self			)    ->    Any:
									"""simple docstring"""







									pass


				def 							A					(				self			)    ->    str:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE	,    SCREAMING_SNAKE_CASE				=	self.model_tester.prepare_config_and_inputs_for_common()

									for model_class in self.all_model_classes:
														if not model_class.supports_gradient_checkpointing:
																			continue

														SCREAMING_SNAKE_CASE				=	True
														SCREAMING_SNAKE_CASE				=	model_class(lowercase__			)
														self.assertTrue(model.is_gradient_checkpointing			)


				def 							A					(				self			)    ->    Optional[Any]:
									"""simple docstring"""







									def check_hidden_states_output(lowercase__		,	lowercase__		,	lowercase__			):
														SCREAMING_SNAKE_CASE				=	model_class(lowercase__			)
														model.to(lowercase__			)
														model.eval()

														with torch.no_grad():
																			SCREAMING_SNAKE_CASE				=	model(**self._prepare_for_class(lowercase__		,	lowercase__			)			)

														SCREAMING_SNAKE_CASE				=	outputs.hidden_states

														SCREAMING_SNAKE_CASE				=	getattr(
														    self.model_tester		,	'expected_num_hidden_layers'		,	self.model_tester.num_hidden_layers + 1			)
														self.assertEqual(len(lowercase__			)		,	lowercase__			)

														self.assertListEqual(
														    list(hidden_states[0].shape[-2:]			)		,	[self.model_tester.patch_embeds_hidden_size, self.model_tester.hidden_size]		,	)

									SCREAMING_SNAKE_CASE	,    SCREAMING_SNAKE_CASE				=	self.model_tester.prepare_config_and_inputs_for_common()

									for model_class in self.all_model_classes:
														SCREAMING_SNAKE_CASE				=	True
														check_hidden_states_output(lowercase__		,	lowercase__		,	lowercase__			)

														# check that output_hidden_states also work using config
														del inputs_dict["output_hidden_states"]
														SCREAMING_SNAKE_CASE				=	True

														check_hidden_states_output(lowercase__		,	lowercase__		,	lowercase__			)


				def 							A					(				self			)    ->    Union[str, Any]:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE	,    SCREAMING_SNAKE_CASE				=	self.model_tester.prepare_config_and_inputs_for_common()

									SCREAMING_SNAKE_CASE				=	_config_zero_init(lowercase__			)
									for model_class in self.all_model_classes:
														SCREAMING_SNAKE_CASE				=	model_class(config=lowercase__			)
														for name, param in model.named_parameters():
																			if isinstance(lowercase__		,	(nn.Linear, nn.Convad, nn.LayerNorm)			):
																								if param.requires_grad:
																													self.assertIn(
																													    ((param.data.mean() * 1E9).round() / 1E9).item()		,	[0.0, 1.0]		,	msg=f'''Parameter {name} of model {model_class} seems not properly initialized'''		,	)




				@unittest.skip(reason='Retain_grad is tested in individual model tests'			)
				def 							A					(				self			)    ->    List[str]:
									"""simple docstring"""







									pass






def   UpperCamelCase_  (		):
					SCREAMING_SNAKE_CASE				=	'https://i.postimg.cc/ZKwLg2Gw/367-14.png'
					SCREAMING_SNAKE_CASE				=	Image.open(requests.get(SCREAMING_SNAKE_CASE_,     stream=SCREAMING_SNAKE_CASE_			).raw			).convert('RGB'			)
					return im




@require_vision
@require_torch
class      UpperCamelCase	(       unittest.TestCase      ):

				"""simple docstring"""



				@slow
				def 							A					(				self			)    ->    List[str]:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE				=	'alibaba-damo/mgp-str-base'
									SCREAMING_SNAKE_CASE				=	MgpstrForSceneTextRecognition.from_pretrained(lowercase__			).to(lowercase__			)
									SCREAMING_SNAKE_CASE				=	MgpstrProcessor.from_pretrained(lowercase__			)

									SCREAMING_SNAKE_CASE				=	prepare_img()
									SCREAMING_SNAKE_CASE				=	processor(images=lowercase__		,	return_tensors='pt'			).pixel_values.to(lowercase__			)

									# forward pass
									with torch.no_grad():
														SCREAMING_SNAKE_CASE				=	model(lowercase__			)

									# verify the logits
									self.assertEqual(outputs.logits[0].shape		,	torch.Size((1, 27, 38)			)			)

									SCREAMING_SNAKE_CASE				=	processor.batch_decode(outputs.logits			)
									SCREAMING_SNAKE_CASE				=	'ticket'

									self.assertEqual(out_strs['generated_text'][0]		,	lowercase__			)

									SCREAMING_SNAKE_CASE				=	torch.tensor(
									    [[[-39.5_397, -44.4_024, -36.1_844], [-61.4_709, -63.8_639, -58.3_454], [-74.0_225, -68.5_494, -71.2_164]]]		,	device=lowercase__		,	)

									self.assertTrue(torch.allclose(outputs.logits[0][:, 1:4, 1:4]		,	lowercase__		,	atol=1E-4			)			)


"""simple docstring"""




import os
from shutil import copyfile
from typing import List, Optional, Tuple

from ...tokenization_utils_fast import AddedToken, PreTrainedTokenizerFast
from ...utils import logging


snake_case    =		logging.get_logger(__name__)

snake_case    =		{'vocab_file': 'spiece.model', 'tokenizer_file': 'tokenizer.json'}

snake_case    =		{
    'vocab_file': {
        'TsinghuaAI/CPM-Generate': 'https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/spiece.model',
    },
    'tokenizer_file': {
        'TsinghuaAI/CPM-Generate': 'https://huggingface.co/TsinghuaAI/CPM-Generate/resolve/main/tokenizer.json',
    },
}




class      UpperCamelCase	(       __magic_name__      ):

				"""simple docstring"""



				def __init__(				self		,	lowercase__=None		,	lowercase__=None		,	lowercase__=False		,	lowercase__=True		,	lowercase__=False		,	lowercase__="<s>"		,	lowercase__="</s>"		,	lowercase__="<unk>"		,	lowercase__="<sep>"		,	lowercase__="<pad>"		,	lowercase__="<cls>"		,	lowercase__="<mask>"		,	lowercase__=["<eop>", "<eod>"]		,	**lowercase__		,	)    ->    int:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE				=	AddedToken(lowercase__		,	lstrip=lowercase__		,	rstrip=lowercase__			) if isinstance(lowercase__		,	lowercase__			) else mask_token

									super().__init__(
									    vocab_file=lowercase__		,	tokenizer_file=lowercase__		,	do_lower_case=lowercase__		,	remove_space=lowercase__		,	keep_accents=lowercase__		,	bos_token=lowercase__		,	eos_token=lowercase__		,	unk_token=lowercase__		,	sep_token=lowercase__		,	pad_token=lowercase__		,	cls_token=lowercase__		,	mask_token=lowercase__		,	additional_special_tokens=lowercase__		,	**lowercase__		,	)

									SCREAMING_SNAKE_CASE				=	3
									SCREAMING_SNAKE_CASE				=	do_lower_case
									SCREAMING_SNAKE_CASE				=	remove_space
									SCREAMING_SNAKE_CASE				=	keep_accents
									SCREAMING_SNAKE_CASE				=	vocab_file
									SCREAMING_SNAKE_CASE				=	False if not self.vocab_file else True

									try:
														import jieba
									except ModuleNotFoundError as error:
														raise error.__class__(
														    'You need to install jieba to use CpmTokenizer or CpmTokenizerFast. '
														    'See https://pypi.org/project/jieba/ for installation.'			)
									SCREAMING_SNAKE_CASE				=	jieba
									SCREAMING_SNAKE_CASE				=	str.maketrans(' \n'		,	'\u2582\u2583'			)


				def 							A					(				self		,	lowercase__		,	lowercase__ = None			)    ->    List[int]:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE				=	[self.sep_token_id]
									SCREAMING_SNAKE_CASE				=	[self.cls_token_id]
									if token_ids_a is None:
														return token_ids_a + sep + cls
									return token_ids_a + sep + token_ids_a + sep + cls


				def 							A					(				self		,	lowercase__		,	lowercase__ = None			)    ->    List[int]:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE				=	[self.sep_token_id]
									SCREAMING_SNAKE_CASE				=	[2]

									if token_ids_a is None:
														return len(token_ids_a + sep			) * [0] + cls_segment_id
									return len(token_ids_a + sep			) * [0] + len(token_ids_a + sep			) * [1] + cls_segment_id


				def 							A					(				self		,	lowercase__		,	lowercase__ = None			)    ->    Tuple[str]:
									"""simple docstring"""







									if not self.can_save_slow_tokenizer:
														raise ValueError(
														    'Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '
														    'tokenizer.'			)

									if not os.path.isdir(lowercase__			):
														logger.error(f'''Vocabulary path ({save_directory}) should be a directory'''			)
														return
									SCREAMING_SNAKE_CASE				=	os.path.join(
									    lowercase__		,	(filename_prefix + '-' if filename_prefix else '') + VOCAB_FILES_NAMES['vocab_file']			)

									if os.path.abspath(self.vocab_file			) != os.path.abspath(lowercase__			):
														copyfile(self.vocab_file		,	lowercase__			)

									return (out_vocab_file,)


				def 							A					(				self		,	lowercase__		,	*lowercase__		,	**lowercase__			)    ->    Any:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE				=	[
									    ' '.join([x.translate(self.translator			) for x in self.jieba.cut(lowercase__		,	cut_all=lowercase__			)]			)
									    for text in batch_text_or_text_pairs
									]
									return super()._batch_encode_plus(lowercase__		,	*lowercase__		,	**lowercase__			)




				def 							A					(				self		,	*lowercase__		,	**lowercase__			)    ->    Optional[int]:
									"""simple docstring"""







									SCREAMING_SNAKE_CASE				=	super()._decode(*lowercase__		,	**lowercase__			)
									SCREAMING_SNAKE_CASE				=	text.replace(' '		,	''			).replace('\u2582'		,	' '			).replace('\u2583'		,	'\n'			)
									return text
