






import unittest

from transformers import GPTNeoXJapaneseConfig, is_torch_available
from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import GPTNeoXJapaneseTokenizer
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
					import torch

					from transformers import GPTNeoXJapaneseForCausalLM, GPTNeoXJapaneseModel
class     lowerCamelCase_					:







				def __init__(    self							,       __lowerCAmelCase							,       __lowerCAmelCase=1_3							,       __lowerCAmelCase=7							,       __lowerCAmelCase=True							,       __lowerCAmelCase=True							,       __lowerCAmelCase=True							,       __lowerCAmelCase=True							,       __lowerCAmelCase=9_9							,       __lowerCAmelCase=3_2							,       __lowerCAmelCase=5							,       __lowerCAmelCase=4							,       __lowerCAmelCase=4							,       __lowerCAmelCase="gelu"							,       __lowerCAmelCase=0.0							,       __lowerCAmelCase=0.1							,       __lowerCAmelCase=True							,       __lowerCAmelCase=5_1_2							,       __lowerCAmelCase=1_6							,       __lowerCAmelCase=2							,       __lowerCAmelCase=0.02							,       __lowerCAmelCase=3							,       __lowerCAmelCase=4							,       __lowerCAmelCase=None							,       ):
								"""simple docstring"""
								__magic_name__		:str   =   parent
								__magic_name__		:Dict   =   batch_size
								__magic_name__		:List[Any]   =   seq_length
								__magic_name__		:int   =   is_training
								__magic_name__		:Optional[int]   =   use_input_mask
								__magic_name__		:Tuple   =   use_token_type_ids
								__magic_name__		:Optional[Any]   =   use_labels
								__magic_name__		:Tuple   =   vocab_size
								__magic_name__		:List[str]   =   hidden_size
								__magic_name__		:str   =   num_hidden_layers
								__magic_name__		:int   =   num_attention_heads
								__magic_name__		:Tuple   =   intermediate_multiple_size
								__magic_name__		:int   =   hidden_act
								__magic_name__		:Optional[int]   =   hidden_dropout
								__magic_name__		:Optional[Any]   =   attention_dropout
								__magic_name__		:Optional[Any]   =   weight_tying
								__magic_name__		:str   =   max_position_embeddings
								__magic_name__		:Any   =   type_vocab_size
								__magic_name__		:Optional[int]   =   type_sequence_label_size
								__magic_name__		:List[str]   =   initializer_range
								__magic_name__		:Dict   =   num_labels
								__magic_name__		:Optional[Any]   =   num_choices
								__magic_name__		:List[Any]   =   scope







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Union[str, Any]   =   ids_tensor([self.batch_size, self.seq_length]							,       self.vocab_size  )

								__magic_name__		:Optional[Any]   =   None
								if self.use_input_mask:
												__magic_name__		:Optional[Any]   =   random_attention_mask([self.batch_size, self.seq_length]  )

								__magic_name__		:List[str]   =   None
								if self.use_labels:
												__magic_name__		:Union[str, Any]   =   ids_tensor([self.batch_size, self.seq_length]							,       self.num_labels  )

								__magic_name__		:Dict   =   self.get_config()

								return config, input_ids, input_mask, token_labels







				def       A						(    self  ):
								"""simple docstring"""
								return GPTNeoXJapaneseConfig(
								    vocab_size=self.vocab_size							,       hidden_size=self.hidden_size							,       num_hidden_layers=self.num_hidden_layers							,       num_attention_heads=self.num_attention_heads							,       intermediate_multiple_size=self.intermediate_multiple_size							,       hidden_act=self.hidden_act							,       hidden_dropout=self.hidden_dropout							,       attention_dropout=self.attention_dropout							,       weight_tying=self.weight_tying							,       max_position_embeddings=self.max_position_embeddings							,       type_vocab_size=self.type_vocab_size							,       is_decoder=__lowerCAmelCase							,       initializer_range=self.initializer_range							,       )







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__  ,					__magic_name__  ,					__magic_name__  ,					__magic_name__		:List[Any]   =   self.prepare_config_and_inputs()

								__magic_name__		:Union[str, Any]   =   True

								return config, input_ids, input_mask, token_labels







				def       A						(    self							,       __lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase  ):
								"""simple docstring"""
								__magic_name__		:str   =   GPTNeoXJapaneseModel(config=__lowerCAmelCase  )
								model.to(__lowerCAmelCase  )
								model.eval()
								__magic_name__		:Tuple   =   model(__lowerCAmelCase							,       attention_mask=__lowerCAmelCase  )
								__magic_name__		:Dict   =   model(__lowerCAmelCase  )
								self.parent.assertEqual(result.last_hidden_state.shape							,       (self.batch_size, self.seq_length, self.hidden_size)  )







				def       A						(    self							,       __lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase  ):
								"""simple docstring"""
								__magic_name__		:str   =   True
								__magic_name__		:Union[str, Any]   =   GPTNeoXJapaneseModel(__lowerCAmelCase  )
								model.to(__lowerCAmelCase  )
								model.eval()
								__magic_name__		:Any   =   model(__lowerCAmelCase							,       attention_mask=__lowerCAmelCase  )
								self.parent.assertEqual(result.last_hidden_state.shape							,       (self.batch_size, self.seq_length, self.hidden_size)  )







				def       A						(    self							,       __lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase  ):
								"""simple docstring"""
								__magic_name__		:str   =   GPTNeoXJapaneseForCausalLM(config=__lowerCAmelCase  )
								model.to(__lowerCAmelCase  )
								model.eval()
								__magic_name__		:List[Any]   =   model(__lowerCAmelCase							,       attention_mask=__lowerCAmelCase							,       labels=__lowerCAmelCase  )
								self.parent.assertEqual(result.logits.shape							,       (self.batch_size, self.seq_length, self.vocab_size)  )







				def       A						(    self							,       __lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase  ):
								"""simple docstring"""
								__magic_name__		:Optional[Any]   =   True
								__magic_name__		:Dict   =   GPTNeoXJapaneseForCausalLM(config=__lowerCAmelCase  )
								model.to(__lowerCAmelCase  )
								model.eval()

								# first forward pass
								__magic_name__		:List[str]   =   model(__lowerCAmelCase							,       attention_mask=__lowerCAmelCase							,       use_cache=__lowerCAmelCase  )
								__magic_name__		:Optional[Any]   =   outputs.past_key_values

								# create hypothetical multiple next token and extent to next_input_ids
								__magic_name__		:str   =   ids_tensor((self.batch_size, 3)							,       config.vocab_size  )
								__magic_name__		:Optional[Any]   =   ids_tensor((self.batch_size, 3)							,       vocab_size=2  )

								# append to next input_ids and
								__magic_name__		:List[str]   =   torch.cat([input_ids, next_tokens]							,       dim=-1  )
								__magic_name__		:Optional[Any]   =   torch.cat([input_mask, next_mask]							,       dim=-1  )

								__magic_name__		:Tuple   =   model(__lowerCAmelCase							,       attention_mask=__lowerCAmelCase							,       output_hidden_states=__lowerCAmelCase  )
								__magic_name__		:int   =   output_from_no_past['''hidden_states'''][0]
								__magic_name__		:Dict   =   model(
								    __lowerCAmelCase							,       attention_mask=__lowerCAmelCase							,       past_key_values=__lowerCAmelCase							,       output_hidden_states=__lowerCAmelCase							,       )['''hidden_states'''][0]

								# select random slice
								__magic_name__		:int   =   ids_tensor((1,)							,       output_from_past.shape[-1]  ).item()
								__magic_name__		:Optional[Any]   =   output_from_no_past[:, -3:, random_slice_idx].detach()
								__magic_name__		:Optional[int]   =   output_from_past[:, :, random_slice_idx].detach()

								self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]  )

								# test that outputs are equal for slice
								self.parent.assertTrue(torch.allclose(__lowerCAmelCase							,       __lowerCAmelCase							,       atol=1E-3  )  )







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Union[str, Any]   =   self.prepare_config_and_inputs()
								__magic_name__  ,					__magic_name__  ,					__magic_name__  ,					__magic_name__		:Optional[int]   =   config_and_inputs
								__magic_name__		:Optional[Any]   =   {'''input_ids''': input_ids, '''attention_mask''': input_mask}
								return config, inputs_dict







@require_torch
class     lowerCamelCase_					(   lowerCamelCase      ,					lowerCamelCase      ,					unittest.TestCase			):
				a__       							=	(GPTNeoXJapaneseModel, GPTNeoXJapaneseForCausalLM) if is_torch_available() else ()
				a__       							=	(GPTNeoXJapaneseForCausalLM,) if is_torch_available() else ()
				a__       							=	(
				    {'''feature-extraction''': GPTNeoXJapaneseModel, '''text-generation''': GPTNeoXJapaneseForCausalLM}
				    if is_torch_available()
				    else {}
				)
				a__       							=	False
				a__       							=	False
				a__       							=	False
				a__       							=	False







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:int   =   GPTNeoXJapaneseModelTester(self  )
								__magic_name__		:Union[str, Any]   =   ConfigTester(self							,       config_class=__lowerCAmelCase							,       hidden_size=3_7  )







				def       A						(    self  ):
								"""simple docstring"""
								self.config_tester.run_common_tests()







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__  ,					__magic_name__  ,					__magic_name__  ,					__magic_name__		:List[str]   =   self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_model(__lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase  )







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__  ,					__magic_name__  ,					__magic_name__  ,					__magic_name__		:int   =   self.model_tester.prepare_config_and_inputs_for_decoder()
								self.model_tester.create_and_check_model_as_decoder(__lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase  )







				def       A						(    self  ):
								"""simple docstring"""
								# This regression test was failing with PyTorch < 1.3
								__magic_name__  ,					__magic_name__  ,					__magic_name__  ,					__magic_name__		:int   =   self.model_tester.prepare_config_and_inputs_for_decoder()

								__magic_name__		:List[str]   =   None

								self.model_tester.create_and_check_model_as_decoder(__lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase  )







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__  ,					__magic_name__  ,					__magic_name__  ,					__magic_name__		:str   =   self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_decoder_model_past_large_inputs(__lowerCAmelCase							,       __lowerCAmelCase							,       __lowerCAmelCase  )







				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Any   =   self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_causal_lm(*__lowerCAmelCase  )







				@slow
				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:Any   =   '''abeja/gpt-neox-japanese-2.7b'''

								__magic_name__		:Union[str, Any]   =   ['''データサイエンティストとは、''', '''100年後に必要とされる会社は、''', '''フルリモートの環境で働くために必要なことは、''', '''国境の長いトンネルを抜けると''', '''美味しい日本食といえば、''']

								__magic_name__		:Union[str, Any]   =   [
								    '''データサイエンティストとは、データを分析し、ビジネスに役立つ知見を導き出す専門家のことです。''',
								    '''100年後に必要とされる会社は、「人」が中心の会社です。''',
								    '''フルリモートの環境で働くために必要なことは、「自分の時間をコントロールする」ことです。''',
								    '''国境の長いトンネルを抜けると、そこは雪国だった。''',
								    '''美味しい日本食といえば、やっぱりお寿司ですよね。''',
								]

								__magic_name__		:Optional[int]   =   GPTNeoXJapaneseTokenizer.from_pretrained(__lowerCAmelCase  )
								__magic_name__		:Union[str, Any]   =   GPTNeoXJapaneseForCausalLM.from_pretrained(__lowerCAmelCase  )

								__magic_name__		:Optional[Any]   =   []
								for prompt in prompts:
												__magic_name__		:Optional[int]   =   tokenizer(__lowerCAmelCase							,       return_tensors='''pt'''  ).input_ids
												__magic_name__		:List[Any]   =   model.generate(__lowerCAmelCase							,       max_length=5_0  )
												__magic_name__		:Dict   =   tokenizer.batch_decode(__lowerCAmelCase							,       skip_special_tokens=__lowerCAmelCase  )
												predicted_outputs += generated_string
								self.assertListEqual(__lowerCAmelCase							,       __lowerCAmelCase  )














import warnings

from ...processing_utils import ProcessorMixin
from ...tokenization_utils_base import BatchEncoding
class     lowerCamelCase_					(   lowerCamelCase			):
				a__       							=	['''image_processor''', '''tokenizer''']
				a__       							=	'''ChineseCLIPImageProcessor'''
				a__       							=	('''BertTokenizer''', '''BertTokenizerFast''')







				def __init__(    self							,       __lowerCAmelCase=None							,       __lowerCAmelCase=None							,       **__lowerCAmelCase  ):
								"""simple docstring"""
								__magic_name__		:Tuple   =   None
								if "feature_extractor" in kwargs:
												warnings.warn(
												    '''The `feature_extractor` argument is deprecated and will be removed in v5, use `image_processor`'''
												    ''' instead.'''							,       __lowerCAmelCase							,       )
												__magic_name__		:Optional[Any]   =   kwargs.pop('''feature_extractor'''  )

								__magic_name__		:Tuple   =   image_processor if image_processor is not None else feature_extractor
								if image_processor is None:
												raise ValueError('''You need to specify an `image_processor`.'''  )
								if tokenizer is None:
												raise ValueError('''You need to specify a `tokenizer`.'''  )

								super().__init__(__lowerCAmelCase							,       __lowerCAmelCase  )
								__magic_name__		:List[Any]   =   self.image_processor







				def __call__(    self							,       __lowerCAmelCase=None							,       __lowerCAmelCase=None							,       __lowerCAmelCase=None							,       **__lowerCAmelCase  ):
								"""simple docstring"""

								if text is None and images is None:
												raise ValueError('''You have to specify either text or images. Both cannot be none.'''  )

								if text is not None:
												__magic_name__		:int   =   self.tokenizer(__lowerCAmelCase							,       return_tensors=__lowerCAmelCase							,       **__lowerCAmelCase  )

								if images is not None:
												__magic_name__		:Dict   =   self.image_processor(__lowerCAmelCase							,       return_tensors=__lowerCAmelCase							,       **__lowerCAmelCase  )

								if text is not None and images is not None:
												__magic_name__		:Union[str, Any]   =   image_features.pixel_values
												return encoding
								elif text is not None:
												return encoding
								else:
												return BatchEncoding(data=dict(**__lowerCAmelCase  )							,       tensor_type=__lowerCAmelCase  )







				def       A						(    self							,       *__lowerCAmelCase							,       **__lowerCAmelCase  ):
								"""simple docstring"""
								return self.tokenizer.batch_decode(*__lowerCAmelCase							,       **__lowerCAmelCase  )







				def       A						(    self							,       *__lowerCAmelCase							,       **__lowerCAmelCase  ):
								"""simple docstring"""
								return self.tokenizer.decode(*__lowerCAmelCase							,       **__lowerCAmelCase  )







				@property
				def       A						(    self  ):
								"""simple docstring"""
								__magic_name__		:List[Any]   =   self.tokenizer.model_input_names
								__magic_name__		:Any   =   self.image_processor.model_input_names
								return list(dict.fromkeys(tokenizer_input_names + image_processor_input_names  )  )







				@property
				def       A						(    self  ):
								"""simple docstring"""
								warnings.warn(
								    '''`feature_extractor_class` is deprecated and will be removed in v5. Use `image_processor_class` instead.'''							,       __lowerCAmelCase							,       )
								return self.image_processor_class






