


"""simple docstring"""






import unittest

from transformers import GPTNeoXJapaneseConfig, is_torch_available
from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import GPTNeoXJapaneseTokenizer
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
					import torch

					from transformers import GPTNeoXJapaneseForCausalLM, GPTNeoXJapaneseModel






class      A:

		"""simple docstring"""





		def __init__( self					,				SCREAMING_SNAKE_CASE__					,				SCREAMING_SNAKE_CASE__=13					,				SCREAMING_SNAKE_CASE__=7					,				SCREAMING_SNAKE_CASE__=True					,				SCREAMING_SNAKE_CASE__=True					,				SCREAMING_SNAKE_CASE__=True					,				SCREAMING_SNAKE_CASE__=True					,				SCREAMING_SNAKE_CASE__=99					,				SCREAMING_SNAKE_CASE__=32					,				SCREAMING_SNAKE_CASE__=5					,				SCREAMING_SNAKE_CASE__=4					,				SCREAMING_SNAKE_CASE__=4					,				SCREAMING_SNAKE_CASE__="gelu"					,				SCREAMING_SNAKE_CASE__=0.0					,				SCREAMING_SNAKE_CASE__=0.1					,				SCREAMING_SNAKE_CASE__=True					,				SCREAMING_SNAKE_CASE__=5_12					,				SCREAMING_SNAKE_CASE__=16					,				SCREAMING_SNAKE_CASE__=2					,				SCREAMING_SNAKE_CASE__=0.0_2					,				SCREAMING_SNAKE_CASE__=3					,				SCREAMING_SNAKE_CASE__=4					,				SCREAMING_SNAKE_CASE__=None					,				)      ->	Dict:

					"""simple docstring"""



					_UpperCamelCase					:Optional[Any]					     =	parent
					_UpperCamelCase					:Union[str, Any]					     =	batch_size
					_UpperCamelCase					:Dict					     =	seq_length
					_UpperCamelCase					:str					     =	is_training
					_UpperCamelCase					:Optional[int]					     =	use_input_mask
					_UpperCamelCase					:Tuple					     =	use_token_type_ids
					_UpperCamelCase					:int					     =	use_labels
					_UpperCamelCase					:List[str]					     =	vocab_size
					_UpperCamelCase					:Optional[Any]					     =	hidden_size
					_UpperCamelCase					:List[str]					     =	num_hidden_layers
					_UpperCamelCase					:str					     =	num_attention_heads
					_UpperCamelCase					:str					     =	intermediate_multiple_size
					_UpperCamelCase					:List[str]					     =	hidden_act
					_UpperCamelCase					:Any					     =	hidden_dropout
					_UpperCamelCase					:Union[str, Any]					     =	attention_dropout
					_UpperCamelCase					:Optional[int]					     =	weight_tying
					_UpperCamelCase					:str					     =	max_position_embeddings
					_UpperCamelCase					:Optional[Any]					     =	type_vocab_size
					_UpperCamelCase					:Optional[int]					     =	type_sequence_label_size
					_UpperCamelCase					:Dict					     =	initializer_range
					_UpperCamelCase					:str					     =	num_labels
					_UpperCamelCase					:Tuple					     =	num_choices
					_UpperCamelCase					:Union[str, Any]					     =	scope
		def   _UpperCamelCase( self   )      ->	Optional[Any]:

					"""simple docstring"""



					_UpperCamelCase					:int					     =	ids_tensor([self.batch_size, self.seq_length]					,				self.vocab_size   )

					_UpperCamelCase					:int					     =	None
					if self.use_input_mask:
								_UpperCamelCase					:Union[str, Any]					     =	random_attention_mask([self.batch_size, self.seq_length]   )

					_UpperCamelCase					:Optional[Any]					     =	None
					if self.use_labels:
								_UpperCamelCase					:Optional[int]					     =	ids_tensor([self.batch_size, self.seq_length]					,				self.num_labels   )

					_UpperCamelCase					:List[Any]					     =	self.get_config()

					return config, input_ids, input_mask, token_labels
		def   _UpperCamelCase( self   )      ->	str:

					"""simple docstring"""



					return GPTNeoXJapaneseConfig(
					    vocab_size=self.vocab_size					,				hidden_size=self.hidden_size					,				num_hidden_layers=self.num_hidden_layers					,				num_attention_heads=self.num_attention_heads					,				intermediate_multiple_size=self.intermediate_multiple_size					,				hidden_act=self.hidden_act					,				hidden_dropout=self.hidden_dropout					,				attention_dropout=self.attention_dropout					,				weight_tying=self.weight_tying					,				max_position_embeddings=self.max_position_embeddings					,				type_vocab_size=self.type_vocab_size					,				is_decoder=SCREAMING_SNAKE_CASE__					,				initializer_range=self.initializer_range					,				)
		def   _UpperCamelCase( self   )      ->	Optional[int]:

					"""simple docstring"""



					_UpperCamelCase     ,      _UpperCamelCase     ,      _UpperCamelCase     ,      _UpperCamelCase					:Any					     =	self.prepare_config_and_inputs()

					_UpperCamelCase					:Any					     =	True

					return config, input_ids, input_mask, token_labels
		def   _UpperCamelCase( self					,				SCREAMING_SNAKE_CASE__					,				SCREAMING_SNAKE_CASE__					,				SCREAMING_SNAKE_CASE__   )      ->	str:

					"""simple docstring"""



					_UpperCamelCase					:List[str]					     =	GPTNeoXJapaneseModel(config=SCREAMING_SNAKE_CASE__   )
					model.to(SCREAMING_SNAKE_CASE__   )
					model.eval()
					_UpperCamelCase					:List[str]					     =	model(SCREAMING_SNAKE_CASE__					,				attention_mask=SCREAMING_SNAKE_CASE__   )
					_UpperCamelCase					:Any					     =	model(SCREAMING_SNAKE_CASE__   )
					self.parent.assertEqual(result.last_hidden_state.shape					,				(self.batch_size, self.seq_length, self.hidden_size)   )
		def   _UpperCamelCase( self					,				SCREAMING_SNAKE_CASE__					,				SCREAMING_SNAKE_CASE__					,				SCREAMING_SNAKE_CASE__   )      ->	Optional[int]:

					"""simple docstring"""



					_UpperCamelCase					:Optional[int]					     =	True
					_UpperCamelCase					:Optional[Any]					     =	GPTNeoXJapaneseModel(SCREAMING_SNAKE_CASE__   )
					model.to(SCREAMING_SNAKE_CASE__   )
					model.eval()
					_UpperCamelCase					:str					     =	model(SCREAMING_SNAKE_CASE__					,				attention_mask=SCREAMING_SNAKE_CASE__   )
					self.parent.assertEqual(result.last_hidden_state.shape					,				(self.batch_size, self.seq_length, self.hidden_size)   )
		def   _UpperCamelCase( self					,				SCREAMING_SNAKE_CASE__					,				SCREAMING_SNAKE_CASE__					,				SCREAMING_SNAKE_CASE__					,				SCREAMING_SNAKE_CASE__   )      ->	str:

					"""simple docstring"""



					_UpperCamelCase					:str					     =	GPTNeoXJapaneseForCausalLM(config=SCREAMING_SNAKE_CASE__   )
					model.to(SCREAMING_SNAKE_CASE__   )
					model.eval()
					_UpperCamelCase					:Tuple					     =	model(SCREAMING_SNAKE_CASE__					,				attention_mask=SCREAMING_SNAKE_CASE__					,				labels=SCREAMING_SNAKE_CASE__   )
					self.parent.assertEqual(result.logits.shape					,				(self.batch_size, self.seq_length, self.vocab_size)   )
		def   _UpperCamelCase( self					,				SCREAMING_SNAKE_CASE__					,				SCREAMING_SNAKE_CASE__					,				SCREAMING_SNAKE_CASE__   )      ->	Dict:

					"""simple docstring"""



					_UpperCamelCase					:Tuple					     =	True
					_UpperCamelCase					:Any					     =	GPTNeoXJapaneseForCausalLM(config=SCREAMING_SNAKE_CASE__   )
					model.to(SCREAMING_SNAKE_CASE__   )
					model.eval()

					# first forward pass
					_UpperCamelCase					:str					     =	model(SCREAMING_SNAKE_CASE__					,				attention_mask=SCREAMING_SNAKE_CASE__					,				use_cache=SCREAMING_SNAKE_CASE__   )
					_UpperCamelCase					:List[Any]					     =	outputs.past_key_values

					# create hypothetical multiple next token and extent to next_input_ids
					_UpperCamelCase					:Any					     =	ids_tensor((self.batch_size, 3)					,				config.vocab_size   )
					_UpperCamelCase					:int					     =	ids_tensor((self.batch_size, 3)					,				vocab_size=2   )

					# append to next input_ids and
					_UpperCamelCase					:str					     =	torch.cat([input_ids, next_tokens]					,				dim=-1   )
					_UpperCamelCase					:Dict					     =	torch.cat([input_mask, next_mask]					,				dim=-1   )

					_UpperCamelCase					:str					     =	model(SCREAMING_SNAKE_CASE__					,				attention_mask=SCREAMING_SNAKE_CASE__					,				output_hidden_states=SCREAMING_SNAKE_CASE__   )
					_UpperCamelCase					:Optional[Any]					     =	output_from_no_past['''hidden_states'''][0]
					_UpperCamelCase					:List[Any]					     =	model(
					    SCREAMING_SNAKE_CASE__					,				attention_mask=SCREAMING_SNAKE_CASE__					,				past_key_values=SCREAMING_SNAKE_CASE__					,				output_hidden_states=SCREAMING_SNAKE_CASE__					,				)['''hidden_states'''][0]

					# select random slice
					_UpperCamelCase					:Dict					     =	ids_tensor((1,)					,				output_from_past.shape[-1]   ).item()
					_UpperCamelCase					:Any					     =	output_from_no_past[:, -3:, random_slice_idx].detach()
					_UpperCamelCase					:List[Any]					     =	output_from_past[:, :, random_slice_idx].detach()

					self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]   )

					# test that outputs are equal for slice
					self.parent.assertTrue(torch.allclose(SCREAMING_SNAKE_CASE__					,				SCREAMING_SNAKE_CASE__					,				atol=1E-3   )   )





		def   _UpperCamelCase( self   )      ->	List[Any]:

					"""simple docstring"""



					_UpperCamelCase					:List[Any]					     =	self.prepare_config_and_inputs()
					_UpperCamelCase     ,      _UpperCamelCase     ,      _UpperCamelCase     ,      _UpperCamelCase					:Any					     =	config_and_inputs
					_UpperCamelCase					:Any					     =	{'''input_ids''': input_ids, '''attention_mask''': input_mask}
					return config, inputs_dict






@require_torch
class      A(  lowerCamelCase__				,							lowerCamelCase__				,							unittest.TestCase		):

		"""simple docstring"""





		A    =							(GPTNeoXJapaneseModel, GPTNeoXJapaneseForCausalLM) if is_torch_available() else ()
		A    =							(GPTNeoXJapaneseForCausalLM,) if is_torch_available() else ()
		A    =							(
		    {"feature-extraction": GPTNeoXJapaneseModel, "text-generation": GPTNeoXJapaneseForCausalLM}
		    if is_torch_available()
		    else {}
		)
		A    =							False
		A    =							False
		A    =							False
		A    =							False
		def   _UpperCamelCase( self   )      ->	Union[str, Any]:

					"""simple docstring"""



					_UpperCamelCase					:Tuple					     =	GPTNeoXJapaneseModelTester(self   )
					_UpperCamelCase					:Any					     =	ConfigTester(self					,				config_class=SCREAMING_SNAKE_CASE__					,				hidden_size=37   )
		def   _UpperCamelCase( self   )      ->	Dict:

					"""simple docstring"""



					self.config_tester.run_common_tests()
		def   _UpperCamelCase( self   )      ->	Any:

					"""simple docstring"""



					_UpperCamelCase     ,      _UpperCamelCase     ,      _UpperCamelCase     ,      _UpperCamelCase					:List[str]					     =	self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_model(SCREAMING_SNAKE_CASE__					,				SCREAMING_SNAKE_CASE__					,				SCREAMING_SNAKE_CASE__   )
		def   _UpperCamelCase( self   )      ->	int:

					"""simple docstring"""



					_UpperCamelCase     ,      _UpperCamelCase     ,      _UpperCamelCase     ,      _UpperCamelCase					:List[str]					     =	self.model_tester.prepare_config_and_inputs_for_decoder()
					self.model_tester.create_and_check_model_as_decoder(SCREAMING_SNAKE_CASE__					,				SCREAMING_SNAKE_CASE__					,				SCREAMING_SNAKE_CASE__   )
		def   _UpperCamelCase( self   )      ->	Optional[int]:

					"""simple docstring"""



					_UpperCamelCase     ,      _UpperCamelCase     ,      _UpperCamelCase     ,      _UpperCamelCase					:Optional[int]					     =	self.model_tester.prepare_config_and_inputs_for_decoder()

					_UpperCamelCase					:Any					     =	None

					self.model_tester.create_and_check_model_as_decoder(SCREAMING_SNAKE_CASE__					,				SCREAMING_SNAKE_CASE__					,				SCREAMING_SNAKE_CASE__   )
		def   _UpperCamelCase( self   )      ->	int:

					"""simple docstring"""



					_UpperCamelCase     ,      _UpperCamelCase     ,      _UpperCamelCase     ,      _UpperCamelCase					:Dict					     =	self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_decoder_model_past_large_inputs(SCREAMING_SNAKE_CASE__					,				SCREAMING_SNAKE_CASE__					,				SCREAMING_SNAKE_CASE__   )
		def   _UpperCamelCase( self   )      ->	List[Any]:

					"""simple docstring"""



					_UpperCamelCase					:str					     =	self.model_tester.prepare_config_and_inputs()
					self.model_tester.create_and_check_for_causal_lm(*SCREAMING_SNAKE_CASE__   )





		@slow
		def   _UpperCamelCase( self   )      ->	Tuple:

					"""simple docstring"""



					_UpperCamelCase					:int					     =	'''abeja/gpt-neox-japanese-2.7b'''

					_UpperCamelCase					:Any					     =	['''データサイエンティストとは、''', '''100年後に必要とされる会社は、''', '''フルリモートの環境で働くために必要なことは、''', '''国境の長いトンネルを抜けると''', '''美味しい日本食といえば、''']

					_UpperCamelCase					:List[str]					     =	[
					    '''データサイエンティストとは、データを分析し、ビジネスに役立つ知見を導き出す専門家のことです。''',
					    '''100年後に必要とされる会社は、「人」が中心の会社です。''',
					    '''フルリモートの環境で働くために必要なことは、「自分の時間をコントロールする」ことです。''',
					    '''国境の長いトンネルを抜けると、そこは雪国だった。''',
					    '''美味しい日本食といえば、やっぱりお寿司ですよね。''',
					]

					_UpperCamelCase					:Optional[Any]					     =	GPTNeoXJapaneseTokenizer.from_pretrained(SCREAMING_SNAKE_CASE__   )
					_UpperCamelCase					:Union[str, Any]					     =	GPTNeoXJapaneseForCausalLM.from_pretrained(SCREAMING_SNAKE_CASE__   )

					_UpperCamelCase					:List[Any]					     =	[]
					for prompt in prompts:
								_UpperCamelCase					:int					     =	tokenizer(SCREAMING_SNAKE_CASE__					,				return_tensors='''pt'''   ).input_ids
								_UpperCamelCase					:Union[str, Any]					     =	model.generate(SCREAMING_SNAKE_CASE__					,				max_length=50   )
								_UpperCamelCase					:Any					     =	tokenizer.batch_decode(SCREAMING_SNAKE_CASE__					,				skip_special_tokens=SCREAMING_SNAKE_CASE__   )
								predicted_outputs += generated_string
					self.assertListEqual(SCREAMING_SNAKE_CASE__					,				SCREAMING_SNAKE_CASE__   )





"""simple docstring"""






import warnings

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding






class      A(  lowerCamelCase__		):

		"""simple docstring"""





		A    =							["image_processor", "tokenizer"]
		A    =							"ViTImageProcessor"
		A    =							("CLIPTokenizer", "CLIPTokenizerFast")
		def __init__( self					,				SCREAMING_SNAKE_CASE__=None					,				SCREAMING_SNAKE_CASE__=None					,				**SCREAMING_SNAKE_CASE__   )      ->	List[str]:

					"""simple docstring"""



					_UpperCamelCase					:List[str]					     =	None
					if "feature_extractor" in kwargs:
								warnings.warn(
								    '''The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`'''
								    ''' instead.'''					,				SCREAMING_SNAKE_CASE__					,				)
								_UpperCamelCase					:List[str]					     =	kwargs.pop('''feature_extractor'''   )

					_UpperCamelCase					:Optional[Any]					     =	image_processor if image_processor is not None else feature_extractor
					if image_processor is None:
								raise ValueError('''You need to specify an `image_processor`.'''   )
					if tokenizer is None:
								raise ValueError('''You need to specify a `tokenizer`.'''   )

					super().__init__(SCREAMING_SNAKE_CASE__					,				SCREAMING_SNAKE_CASE__   )
		def __call__( self					,				SCREAMING_SNAKE_CASE__=None					,				SCREAMING_SNAKE_CASE__=None					,				SCREAMING_SNAKE_CASE__=None					,				SCREAMING_SNAKE_CASE__=None					,				**SCREAMING_SNAKE_CASE__   )      ->	Tuple:

					"""simple docstring"""



					if text is None and visual_prompt is None and images is None:
								raise ValueError('''You have to specify either text, visual prompt or images.'''   )

					if text is not None and visual_prompt is not None:
								raise ValueError('''You have to specify exactly one type of prompt. Either text or visual prompt.'''   )

					if text is not None:
								_UpperCamelCase					:Dict					     =	self.tokenizer(SCREAMING_SNAKE_CASE__					,				return_tensors=SCREAMING_SNAKE_CASE__					,				**SCREAMING_SNAKE_CASE__   )

					if visual_prompt is not None:
								_UpperCamelCase					:Optional[Any]					     =	self.image_processor(SCREAMING_SNAKE_CASE__					,				return_tensors=SCREAMING_SNAKE_CASE__					,				**SCREAMING_SNAKE_CASE__   )

					if images is not None:
								_UpperCamelCase					:Union[str, Any]					     =	self.image_processor(SCREAMING_SNAKE_CASE__					,				return_tensors=SCREAMING_SNAKE_CASE__					,				**SCREAMING_SNAKE_CASE__   )

					if visual_prompt is not None and images is not None:
								_UpperCamelCase					:List[str]					     =	{
								    '''pixel_values''': image_features.pixel_values,
								    '''conditional_pixel_values''': prompt_features.pixel_values,
								}
								return encoding
					elif text is not None and images is not None:
								_UpperCamelCase					:List[Any]					     =	image_features.pixel_values
								return encoding
					elif text is not None:
								return encoding
					elif visual_prompt is not None:
								_UpperCamelCase					:Optional[Any]					     =	{
								    '''conditional_pixel_values''': prompt_features.pixel_values,
								}
								return encoding
					else:
								return BatchEncoding(data=dict(**SCREAMING_SNAKE_CASE__   )					,				tensor_type=SCREAMING_SNAKE_CASE__   )
		def   _UpperCamelCase( self					,				*SCREAMING_SNAKE_CASE__					,				**SCREAMING_SNAKE_CASE__   )      ->	Any:

					"""simple docstring"""



					return self.tokenizer.batch_decode(*SCREAMING_SNAKE_CASE__					,				**SCREAMING_SNAKE_CASE__   )
		def   _UpperCamelCase( self					,				*SCREAMING_SNAKE_CASE__					,				**SCREAMING_SNAKE_CASE__   )      ->	Tuple:

					"""simple docstring"""



					return self.tokenizer.decode(*SCREAMING_SNAKE_CASE__					,				**SCREAMING_SNAKE_CASE__   )
		@property
		def   _UpperCamelCase( self   )      ->	Tuple:

					"""simple docstring"""



					warnings.warn(
					    '''`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead.'''					,				SCREAMING_SNAKE_CASE__					,				)
					return self.image_processor_class





		@property
		def   _UpperCamelCase( self   )      ->	Union[str, Any]:

					"""simple docstring"""



					warnings.warn(
					    '''`feature_extractor` is deprecated and will be removed in v5. Use `image_processor` instead.'''					,				SCREAMING_SNAKE_CASE__					,				)
					return self.image_processor

