


'''simple docstring'''
import unittest

from transformers import GPTNeoXJapaneseConfig, is_torch_available
from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import GPTNeoXJapaneseTokenizer
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
		import torch

		from transformers import GPTNeoXJapaneseForCausalLM, GPTNeoXJapaneseModel
class   lowercase      :

		"""simple docstring"""

		def __init__(						self       ,  UpperCamelCase_       ,  UpperCamelCase_=13       ,  UpperCamelCase_=7       ,  UpperCamelCase_=True       ,  UpperCamelCase_=True       ,  UpperCamelCase_=True       ,  UpperCamelCase_=True       ,  UpperCamelCase_=99       ,  UpperCamelCase_=32       ,  UpperCamelCase_=5       ,  UpperCamelCase_=4       ,  UpperCamelCase_=4       ,  UpperCamelCase_="gelu"       ,  UpperCamelCase_=0.0       ,  UpperCamelCase_=0.1       ,  UpperCamelCase_=True       ,  UpperCamelCase_=512       ,  UpperCamelCase_=16       ,  UpperCamelCase_=2       ,  UpperCamelCase_=0.02       ,  UpperCamelCase_=3       ,  UpperCamelCase_=4       ,  UpperCamelCase_=None       ,  ):

							'''simple docstring'''


							UpperCamelCase__  :Tuple				   =					parent
							UpperCamelCase__  :int				   =					batch_size
							UpperCamelCase__  :Tuple				   =					seq_length
							UpperCamelCase__  :List[str]				   =					is_training
							UpperCamelCase__  :List[Any]				   =					use_input_mask
							UpperCamelCase__  :str				   =					use_token_type_ids
							UpperCamelCase__  :Optional[Any]				   =					use_labels
							UpperCamelCase__  :List[Any]				   =					vocab_size
							UpperCamelCase__  :str				   =					hidden_size
							UpperCamelCase__  :List[str]				   =					num_hidden_layers
							UpperCamelCase__  :int				   =					num_attention_heads
							UpperCamelCase__  :Optional[int]				   =					intermediate_multiple_size
							UpperCamelCase__  :List[Any]				   =					hidden_act
							UpperCamelCase__  :Any				   =					hidden_dropout
							UpperCamelCase__  :Optional[Any]				   =					attention_dropout
							UpperCamelCase__  :List[Any]				   =					weight_tying
							UpperCamelCase__  :Optional[int]				   =					max_position_embeddings
							UpperCamelCase__  :Any				   =					type_vocab_size
							UpperCamelCase__  :Dict				   =					type_sequence_label_size
							UpperCamelCase__  :List[Any]				   =					initializer_range
							UpperCamelCase__  :Union[str, Any]				   =					num_labels
							UpperCamelCase__  :Union[str, Any]				   =					num_choices
							UpperCamelCase__  :Tuple				   =					scope




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__  :Optional[Any]				   =					ids_tensor([self.batch_size, self.seq_length]       ,  self.vocab_size				)

							UpperCamelCase__  :Optional[Any]				   =					None
							if self.use_input_mask:
												UpperCamelCase__  :Optional[Any]				   =					random_attention_mask([self.batch_size, self.seq_length]				)

							UpperCamelCase__  :Tuple				   =					None
							if self.use_labels:
												UpperCamelCase__  :List[str]				   =					ids_tensor([self.batch_size, self.seq_length]       ,  self.num_labels				)

							UpperCamelCase__  :List[str]				   =					self.get_config()

							return config, input_ids, input_mask, token_labels




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							return GPTNeoXJapaneseConfig(
							    vocab_size=self.vocab_size       ,  hidden_size=self.hidden_size       ,  num_hidden_layers=self.num_hidden_layers       ,  num_attention_heads=self.num_attention_heads       ,  intermediate_multiple_size=self.intermediate_multiple_size       ,  hidden_act=self.hidden_act       ,  hidden_dropout=self.hidden_dropout       ,  attention_dropout=self.attention_dropout       ,  weight_tying=self.weight_tying       ,  max_position_embeddings=self.max_position_embeddings       ,  type_vocab_size=self.type_vocab_size       ,  is_decoder=UpperCamelCase_       ,  initializer_range=self.initializer_range       ,  )




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__   ,							UpperCamelCase__   ,							UpperCamelCase__   ,							UpperCamelCase__  :Optional[Any]				   =					self.prepare_config_and_inputs()

							UpperCamelCase__  :Dict				   =					True

							return config, input_ids, input_mask, token_labels




		def        lowerCAmelCase__							(						self       ,  UpperCamelCase_       ,  UpperCamelCase_       ,  UpperCamelCase_				):

							'''simple docstring'''


							UpperCamelCase__  :int				   =					GPTNeoXJapaneseModel(config=UpperCamelCase_				)
							model.to(UpperCamelCase_				)
							model.eval()
							UpperCamelCase__  :Dict				   =					model(UpperCamelCase_       ,  attention_mask=UpperCamelCase_				)
							UpperCamelCase__  :Optional[Any]				   =					model(UpperCamelCase_				)
							self.parent.assertEqual(result.last_hidden_state.shape       ,  (self.batch_size, self.seq_length, self.hidden_size)				)




		def        lowerCAmelCase__							(						self       ,  UpperCamelCase_       ,  UpperCamelCase_       ,  UpperCamelCase_				):

							'''simple docstring'''


							UpperCamelCase__  :int				   =					True
							UpperCamelCase__  :Union[str, Any]				   =					GPTNeoXJapaneseModel(UpperCamelCase_				)
							model.to(UpperCamelCase_				)
							model.eval()
							UpperCamelCase__  :Any				   =					model(UpperCamelCase_       ,  attention_mask=UpperCamelCase_				)
							self.parent.assertEqual(result.last_hidden_state.shape       ,  (self.batch_size, self.seq_length, self.hidden_size)				)




		def        lowerCAmelCase__							(						self       ,  UpperCamelCase_       ,  UpperCamelCase_       ,  UpperCamelCase_       ,  UpperCamelCase_				):

							'''simple docstring'''


							UpperCamelCase__  :Union[str, Any]				   =					GPTNeoXJapaneseForCausalLM(config=UpperCamelCase_				)
							model.to(UpperCamelCase_				)
							model.eval()
							UpperCamelCase__  :str				   =					model(UpperCamelCase_       ,  attention_mask=UpperCamelCase_       ,  labels=UpperCamelCase_				)
							self.parent.assertEqual(result.logits.shape       ,  (self.batch_size, self.seq_length, self.vocab_size)				)




		def        lowerCAmelCase__							(						self       ,  UpperCamelCase_       ,  UpperCamelCase_       ,  UpperCamelCase_				):

							'''simple docstring'''


							UpperCamelCase__  :Dict				   =					True
							UpperCamelCase__  :str				   =					GPTNeoXJapaneseForCausalLM(config=UpperCamelCase_				)
							model.to(UpperCamelCase_				)
							model.eval()

							# first forward pass
							UpperCamelCase__  :Optional[int]				   =					model(UpperCamelCase_       ,  attention_mask=UpperCamelCase_       ,  use_cache=UpperCamelCase_				)
							UpperCamelCase__  :Optional[int]				   =					outputs.past_key_values

							# create hypothetical multiple next token and extent to next_input_ids
							UpperCamelCase__  :Any				   =					ids_tensor((self.batch_size, 3)       ,  config.vocab_size				)
							UpperCamelCase__  :Any				   =					ids_tensor((self.batch_size, 3)       ,  vocab_size=2				)

							# append to next input_ids and
							UpperCamelCase__  :Optional[Any]				   =					torch.cat([input_ids, next_tokens]       ,  dim=-1				)
							UpperCamelCase__  :List[str]				   =					torch.cat([input_mask, next_mask]       ,  dim=-1				)

							UpperCamelCase__  :Tuple				   =					model(UpperCamelCase_       ,  attention_mask=UpperCamelCase_       ,  output_hidden_states=UpperCamelCase_				)
							UpperCamelCase__  :int				   =					output_from_no_past['''hidden_states'''][0]
							UpperCamelCase__  :str				   =					model(
							    UpperCamelCase_       ,  attention_mask=UpperCamelCase_       ,  past_key_values=UpperCamelCase_       ,  output_hidden_states=UpperCamelCase_       ,  )['''hidden_states'''][0]

							# select random slice
							UpperCamelCase__  :List[str]				   =					ids_tensor((1,)       ,  output_from_past.shape[-1]				).item()
							UpperCamelCase__  :List[str]				   =					output_from_no_past[:, -3:, random_slice_idx].detach()
							UpperCamelCase__  :Union[str, Any]				   =					output_from_past[:, :, random_slice_idx].detach()

							self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]				)

							# test that outputs are equal for slice
							self.parent.assertTrue(torch.allclose(UpperCamelCase_       ,  UpperCamelCase_       ,  atol=1e-3				)				)




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__  :Tuple				   =					self.prepare_config_and_inputs()
							UpperCamelCase__   ,							UpperCamelCase__   ,							UpperCamelCase__   ,							UpperCamelCase__  :Union[str, Any]				   =					config_and_inputs
							UpperCamelCase__  :Any				   =					{'''input_ids''': input_ids, '''attention_mask''': input_mask}
							return config, inputs_dict



@require_torch
class   lowercase      (				A__      ,						A__      ,						unittest.TestCase   ):

		"""simple docstring"""

		_a       					=							(GPTNeoXJapaneseModel, GPTNeoXJapaneseForCausalLM) if is_torch_available() else ()
		_a       					=							(GPTNeoXJapaneseForCausalLM,) if is_torch_available() else ()
		_a       					=							(
		    {'feature-extraction': GPTNeoXJapaneseModel, 'text-generation': GPTNeoXJapaneseForCausalLM}
		    if is_torch_available()
		    else {}
		)
		_a       					=							False
		_a       					=							False
		_a       					=							False
		_a       					=							False




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__  :Optional[int]				   =					GPTNeoXJapaneseModelTester(self				)
							UpperCamelCase__  :Optional[Any]				   =					ConfigTester(self       ,  config_class=UpperCamelCase_       ,  hidden_size=37				)




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							self.config_tester.run_common_tests()




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__   ,							UpperCamelCase__   ,							UpperCamelCase__   ,							UpperCamelCase__  :Tuple				   =					self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_model(UpperCamelCase_       ,  UpperCamelCase_       ,  UpperCamelCase_				)




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__   ,							UpperCamelCase__   ,							UpperCamelCase__   ,							UpperCamelCase__  :List[str]				   =					self.model_tester.prepare_config_and_inputs_for_decoder()
							self.model_tester.create_and_check_model_as_decoder(UpperCamelCase_       ,  UpperCamelCase_       ,  UpperCamelCase_				)




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__   ,							UpperCamelCase__   ,							UpperCamelCase__   ,							UpperCamelCase__  :int				   =					self.model_tester.prepare_config_and_inputs_for_decoder()

							UpperCamelCase__  :Union[str, Any]				   =					None

							self.model_tester.create_and_check_model_as_decoder(UpperCamelCase_       ,  UpperCamelCase_       ,  UpperCamelCase_				)




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__   ,							UpperCamelCase__   ,							UpperCamelCase__   ,							UpperCamelCase__  :str				   =					self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_decoder_model_past_large_inputs(UpperCamelCase_       ,  UpperCamelCase_       ,  UpperCamelCase_				)




		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__  :Tuple				   =					self.model_tester.prepare_config_and_inputs()
							self.model_tester.create_and_check_for_causal_lm(*UpperCamelCase_				)




		@slow
		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__  :List[Any]				   =					'''abeja/gpt-neox-japanese-2.7b'''

							UpperCamelCase__  :Union[str, Any]				   =					['''データサイエンティストとは、''', '''100年後に必要とされる会社は、''', '''フルリモートの環境で働くために必要なことは、''', '''国境の長いトンネルを抜けると''', '''美味しい日本食といえば、''']

							UpperCamelCase__  :Union[str, Any]				   =					[
							    '''データサイエンティストとは、データを分析し、ビジネスに役立つ知見を導き出す専門家のことです。''',
							    '''100年後に必要とされる会社は、「人」が中心の会社です。''',
							    '''フルリモートの環境で働くために必要なことは、「自分の時間をコントロールする」ことです。''',
							    '''国境の長いトンネルを抜けると、そこは雪国だった。''',
							    '''美味しい日本食といえば、やっぱりお寿司ですよね。''',
							]

							UpperCamelCase__  :Optional[Any]				   =					GPTNeoXJapaneseTokenizer.from_pretrained(UpperCamelCase_				)
							UpperCamelCase__  :List[str]				   =					GPTNeoXJapaneseForCausalLM.from_pretrained(UpperCamelCase_				)

							UpperCamelCase__  :Any				   =					[]
							for prompt in prompts:
												UpperCamelCase__  :Any				   =					tokenizer(UpperCamelCase_       ,  return_tensors='''pt'''				).input_ids
												UpperCamelCase__  :List[str]				   =					model.generate(UpperCamelCase_       ,  max_length=50				)
												UpperCamelCase__  :str				   =					tokenizer.batch_decode(UpperCamelCase_       ,  skip_special_tokens=UpperCamelCase_				)
												predicted_outputs += generated_string
							self.assertListEqual(UpperCamelCase_       ,  UpperCamelCase_				)



'''simple docstring'''
import warnings

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding
class   lowercase      (				A__   ):

		"""simple docstring"""

		_a       					=							['image_processor', 'tokenizer']
		_a       					=							'ChineseCLIPImageProcessor'
		_a       					=							('BertTokenizer', 'BertTokenizerFast')




		def __init__(						self       ,  UpperCamelCase_=None       ,  UpperCamelCase_=None       ,  **UpperCamelCase_				):

							'''simple docstring'''


							UpperCamelCase__  :List[Any]				   =					None
							if "feature_extractor" in kwargs:
												warnings.warn(
												    '''The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`'''
												    ''' instead.'''       ,  UpperCamelCase_       ,  )
												UpperCamelCase__  :List[Any]				   =					kwargs.pop('''feature_extractor'''				)

							UpperCamelCase__  :List[str]				   =					image_processor if image_processor is not None else feature_extractor
							if image_processor is None:
												raise ValueError('''You need to specify an `image_processor`.'''				)
							if tokenizer is None:
												raise ValueError('''You need to specify a `tokenizer`.'''				)

							super().__init__(UpperCamelCase_       ,  UpperCamelCase_				)
							UpperCamelCase__  :Any				   =					self.image_processor




		def __call__(						self       ,  UpperCamelCase_=None       ,  UpperCamelCase_=None       ,  UpperCamelCase_=None       ,  **UpperCamelCase_				):

							'''simple docstring'''


							if text is None and images is None:
												raise ValueError('''You have to specify either text or images. Both cannot be none.'''				)

							if text is not None:
												UpperCamelCase__  :Tuple				   =					self.tokenizer(UpperCamelCase_       ,  return_tensors=UpperCamelCase_       ,  **UpperCamelCase_				)

							if images is not None:
												UpperCamelCase__  :Any				   =					self.image_processor(UpperCamelCase_       ,  return_tensors=UpperCamelCase_       ,  **UpperCamelCase_				)

							if text is not None and images is not None:
												UpperCamelCase__  :Optional[int]				   =					image_features.pixel_values
												return encoding
							elif text is not None:
												return encoding
							else:
												return BatchEncoding(data=dict(**UpperCamelCase_				)       ,  tensor_type=UpperCamelCase_				)




		def        lowerCAmelCase__							(						self       ,  *UpperCamelCase_       ,  **UpperCamelCase_				):

							'''simple docstring'''


							return self.tokenizer.batch_decode(*UpperCamelCase_       ,  **UpperCamelCase_				)




		def        lowerCAmelCase__							(						self       ,  *UpperCamelCase_       ,  **UpperCamelCase_				):

							'''simple docstring'''


							return self.tokenizer.decode(*UpperCamelCase_       ,  **UpperCamelCase_				)




		@property
		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							UpperCamelCase__  :int				   =					self.tokenizer.model_input_names
							UpperCamelCase__  :Any				   =					self.image_processor.model_input_names
							return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names				)				)




		@property
		def        lowerCAmelCase__							(						self				):

							'''simple docstring'''


							warnings.warn(
							    '''`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead.'''       ,  UpperCamelCase_       ,  )
							return self.image_processor_class