"""simple docstring"""







import os
import sys
import tempfile
import unittest
import unittest.mock as mock
from pathlib import Path

from huggingface_hub import HfFolder, delete_repo
from huggingface_hub.file_download import http_get
from requests.exceptions import HTTPError

from transformers import (
    AlbertTokenizer,
    AutoTokenizer,
    BertTokenizer,
    BertTokenizerFast,
    GPTaTokenizerFast,
    is_tokenizers_available,
)
from transformers.testing_utils import TOKEN, USER, is_staging_test, require_tokenizers
from transformers.tokenization_utils import Trie


sys.path.append(str(Path(__file__).parent.parent / "utils"))

from test_module.custom_tokenization import CustomTokenizer  # noqa E402


if is_tokenizers_available():
       from test_module.custom_tokenization_fast import CustomTokenizerFast







class 			UpperCamelCase						(     unittest.TestCase    ):
   def      _UpperCAmelCase  (     self     )      -> Tuple:


        '''simple docstring'''


        lowercase_      :							str						=      mock.Mock()
        lowercase_      :							Optional[Any]						=      500
        lowercase_      :							List[str]						=      {}
        lowercase_      :							Optional[Any]						=      HTTPError
        lowercase_      :							Union[str, Any]						=      {}

        # Download this model to make sure it's in the cache.
        lowercase_      :							int						=      BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'     )

        # Under the mock environment we get a 500 error when trying to reach the tokenizer.
        with mock.patch('requests.Session.request'	,return_value=UpperCAmelCase_     ) as mock_head:
             lowercase_      :							Dict						=      BertTokenizer.from_pretrained('hf-internal-testing/tiny-random-bert'     )
             # This check we did call the fake head request
             mock_head.assert_called()
   @require_tokenizers
   def      _UpperCAmelCase  (     self     )      -> Tuple:


        '''simple docstring'''


        lowercase_      :							Tuple						=      mock.Mock()
        lowercase_      :							Optional[int]						=      500
        lowercase_      :							Any						=      {}
        lowercase_      :							List[str]						=      HTTPError
        lowercase_      :							List[Any]						=      {}

        # Download this model to make sure it's in the cache.
        lowercase_      :							str						=      GPTaTokenizerFast.from_pretrained('gpt2'     )

        # Under the mock environment we get a 500 error when trying to reach the tokenizer.
        with mock.patch('requests.Session.request'	,return_value=UpperCAmelCase_     ) as mock_head:
             lowercase_      :							List[Any]						=      GPTaTokenizerFast.from_pretrained('gpt2'     )
             # This check we did call the fake head request
             mock_head.assert_called()
   def      _UpperCAmelCase  (     self     )      -> str:


        '''simple docstring'''


        try:
             lowercase_      :							List[Any]						=      tempfile.mktemp()
             with open(UpperCAmelCase_	,'wb'     ) as f:
                  http_get('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'	,UpperCAmelCase_     )

             lowercase_      :							int						=      AlbertTokenizer.from_pretrained(UpperCAmelCase_     )
        finally:
             os.remove(UpperCAmelCase_     )

        # Supporting this legacy load introduced a weird bug where the tokenizer would load local files if they are in
        # the current folder and have the right name.
        if os.path.isfile('tokenizer.json'     ):
             # We skip the test if the user has a `tokenizer.json` in this folder to avoid deleting it.
             return
        try:
             with open('tokenizer.json'	,'wb'     ) as f:
                  http_get('https://huggingface.co/hf-internal-testing/tiny-random-bert/blob/main/tokenizer.json'	,UpperCAmelCase_     )
             lowercase_      :							Tuple						=      AutoTokenizer.from_pretrained('hf-internal-testing/tiny-random-gpt2'     )
             # The tiny random BERT has a vocab size of 1024, tiny gpt2 as a vocab size of 1000
             self.assertEqual(tokenizer.vocab_size	,1000     )
             # Tokenizer should depend on the remote checkpoint, not the local tokenizer.json file.

        finally:
             os.remove('tokenizer.json'     )







   def      _UpperCAmelCase  (     self     )      -> Any:


        '''simple docstring'''


        lowercase_      :							List[str]						=      AlbertTokenizer.from_pretrained('https://huggingface.co/albert-base-v1/resolve/main/spiece.model'     )







@is_staging_test
class 			UpperCamelCase						(     unittest.TestCase    ):
   lowercase				=    ['''[UNK]''', '''[CLS]''', '''[SEP]''', '''[PAD]''', '''[MASK]''', '''bla''', '''blou''']
   @classmethod
   def      _UpperCAmelCase  (     cls     )      -> Any:


        '''simple docstring'''


        lowercase_      :							Dict						=      TOKEN
        HfFolder.save_token(UpperCAmelCase_     )
   @classmethod
   def      _UpperCAmelCase  (     cls     )      -> Any:


        '''simple docstring'''


        try:
             delete_repo(token=cls._token	,repo_id='test-tokenizer'     )
        except HTTPError:
             pass

        try:
             delete_repo(token=cls._token	,repo_id='valid_org/test-tokenizer-org'     )
        except HTTPError:
             pass

        try:
             delete_repo(token=cls._token	,repo_id='test-dynamic-tokenizer'     )
        except HTTPError:
             pass
   def      _UpperCAmelCase  (     self     )      -> Optional[Any]:


        '''simple docstring'''


        with tempfile.TemporaryDirectory() as tmp_dir:
             lowercase_      :							Optional[int]						=      os.path.join(UpperCAmelCase_	,'vocab.txt'     )
             with open(UpperCAmelCase_	,'w'	,encoding='utf-8'     ) as vocab_writer:
                  vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]     )     )
             lowercase_      :							Dict						=      BertTokenizer(UpperCAmelCase_     )

        tokenizer.push_to_hub('test-tokenizer'	,use_auth_token=self._token     )
        lowercase_      :							str						=      BertTokenizer.from_pretrained(f'''{USER}/test-tokenizer'''     )
        self.assertDictEqual(new_tokenizer.vocab	,tokenizer.vocab     )

        # Reset repo
        delete_repo(token=self._token	,repo_id='test-tokenizer'     )

        # Push to hub via save_pretrained
        with tempfile.TemporaryDirectory() as tmp_dir:
             tokenizer.save_pretrained(UpperCAmelCase_	,repo_id='test-tokenizer'	,push_to_hub=UpperCAmelCase_	,use_auth_token=self._token     )

        lowercase_      :							List[Any]						=      BertTokenizer.from_pretrained(f'''{USER}/test-tokenizer'''     )
        self.assertDictEqual(new_tokenizer.vocab	,tokenizer.vocab     )
   def      _UpperCAmelCase  (     self     )      -> Tuple:


        '''simple docstring'''


        with tempfile.TemporaryDirectory() as tmp_dir:
             lowercase_      :							List[Any]						=      os.path.join(UpperCAmelCase_	,'vocab.txt'     )
             with open(UpperCAmelCase_	,'w'	,encoding='utf-8'     ) as vocab_writer:
                  vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]     )     )
             lowercase_      :							str						=      BertTokenizer(UpperCAmelCase_     )

        tokenizer.push_to_hub('valid_org/test-tokenizer-org'	,use_auth_token=self._token     )
        lowercase_      :							Union[str, Any]						=      BertTokenizer.from_pretrained('valid_org/test-tokenizer-org'     )
        self.assertDictEqual(new_tokenizer.vocab	,tokenizer.vocab     )

        # Reset repo
        delete_repo(token=self._token	,repo_id='valid_org/test-tokenizer-org'     )

        # Push to hub via save_pretrained
        with tempfile.TemporaryDirectory() as tmp_dir:
             tokenizer.save_pretrained(
                 UpperCAmelCase_	,repo_id='valid_org/test-tokenizer-org'	,push_to_hub=UpperCAmelCase_	,use_auth_token=self._token     )

        lowercase_      :							Union[str, Any]						=      BertTokenizer.from_pretrained('valid_org/test-tokenizer-org'     )
        self.assertDictEqual(new_tokenizer.vocab	,tokenizer.vocab     )







   @require_tokenizers
   def      _UpperCAmelCase  (     self     )      -> List[Any]:


        '''simple docstring'''


        CustomTokenizer.register_for_auto_class()
        with tempfile.TemporaryDirectory() as tmp_dir:
             lowercase_      :							List[str]						=      os.path.join(UpperCAmelCase_	,'vocab.txt'     )
             with open(UpperCAmelCase_	,'w'	,encoding='utf-8'     ) as vocab_writer:
                  vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]     )     )
             lowercase_      :							List[Any]						=      CustomTokenizer(UpperCAmelCase_     )

        # No fast custom tokenizer
        tokenizer.push_to_hub('test-dynamic-tokenizer'	,use_auth_token=self._token     )

        lowercase_      :							Any						=      AutoTokenizer.from_pretrained(f'''{USER}/test-dynamic-tokenizer'''	,trust_remote_code=UpperCAmelCase_     )
        # Can't make an isinstance check because the new_model.config is from the CustomTokenizer class of a dynamic module
        self.assertEqual(tokenizer.__class__.__name__	,'CustomTokenizer'     )

        # Fast and slow custom tokenizer
        CustomTokenizerFast.register_for_auto_class()
        with tempfile.TemporaryDirectory() as tmp_dir:
             lowercase_      :							List[str]						=      os.path.join(UpperCAmelCase_	,'vocab.txt'     )
             with open(UpperCAmelCase_	,'w'	,encoding='utf-8'     ) as vocab_writer:
                  vocab_writer.write(''.join([x + '\n' for x in self.vocab_tokens]     )     )

             lowercase_      :							Optional[Any]						=      BertTokenizerFast.from_pretrained(UpperCAmelCase_     )
             bert_tokenizer.save_pretrained(UpperCAmelCase_     )
             lowercase_      :							Dict						=      CustomTokenizerFast.from_pretrained(UpperCAmelCase_     )

        tokenizer.push_to_hub('test-dynamic-tokenizer'	,use_auth_token=self._token     )

        lowercase_      :							Optional[Any]						=      AutoTokenizer.from_pretrained(f'''{USER}/test-dynamic-tokenizer'''	,trust_remote_code=UpperCAmelCase_     )
        # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
        self.assertEqual(tokenizer.__class__.__name__	,'CustomTokenizerFast'     )
        lowercase_      :							Tuple						=      AutoTokenizer.from_pretrained(
            f'''{USER}/test-dynamic-tokenizer'''	,use_fast=UpperCAmelCase_	,trust_remote_code=UpperCAmelCase_     )
        # Can't make an isinstance check because the new_model.config is from the FakeConfig class of a dynamic module
        self.assertEqual(tokenizer.__class__.__name__	,'CustomTokenizer'     )







class 			UpperCamelCase						(     unittest.TestCase    ):
   def      _UpperCAmelCase  (     self     )      -> str:


        '''simple docstring'''


        lowercase_      :							Optional[int]						=      Trie()
        trie.add('Hello 友達'     )
        self.assertEqual(trie.data	,{'H': {'e': {'l': {'l': {'o': {' ': {'友': {'達': {'': 1}}}}}}}}}     )
        trie.add('Hello'     )
        trie.data
        self.assertEqual(trie.data	,{'H': {'e': {'l': {'l': {'o': {'': 1, ' ': {'友': {'達': {'': 1}}}}}}}}}     )
   def      _UpperCAmelCase  (     self     )      -> Optional[int]:


        '''simple docstring'''


        lowercase_      :							Dict						=      Trie()
        self.assertEqual(trie.split('[CLS] This is a extra_id_100'     )	,['[CLS] This is a extra_id_100']     )
        trie.add('[CLS]'     )
        trie.add('extra_id_1'     )
        trie.add('extra_id_100'     )
        self.assertEqual(trie.split('[CLS] This is a extra_id_100'     )	,['[CLS]', ' This is a ', 'extra_id_100']     )
   def      _UpperCAmelCase  (     self     )      -> Dict:


        '''simple docstring'''


        lowercase_      :							str						=      Trie()
        trie.add('A'     )
        self.assertEqual(trie.split('ABC'     )	,['A', 'BC']     )
        self.assertEqual(trie.split('BCA'     )	,['BC', 'A']     )
   def      _UpperCAmelCase  (     self     )      -> List[Any]:


        '''simple docstring'''


        lowercase_      :							Dict						=      Trie()
        trie.add('TOKEN]'     )
        trie.add('[SPECIAL_TOKEN]'     )
        self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]'     )	,['This is something ', '[SPECIAL_TOKEN]']     )
   def      _UpperCAmelCase  (     self     )      -> Dict:


        '''simple docstring'''


        lowercase_      :							int						=      Trie()
        trie.add('A'     )
        trie.add('P'     )
        trie.add('[SPECIAL_TOKEN]'     )
        self.assertEqual(trie.split('This is something [SPECIAL_TOKEN]'     )	,['This is something ', '[SPECIAL_TOKEN]']     )
   def      _UpperCAmelCase  (     self     )      -> Any:


        '''simple docstring'''


        lowercase_      :							str						=      Trie()
        trie.add('AB'     )
        trie.add('B'     )
        trie.add('C'     )
        self.assertEqual(trie.split('ABC'     )	,['AB', 'C']     )
   def      _UpperCAmelCase  (     self     )      -> Any:


        '''simple docstring'''


        lowercase_      :							Optional[int]						=      Trie()
        trie.add('ABC'     )
        trie.add('B'     )
        trie.add('CD'     )
        self.assertEqual(trie.split('ABCD'     )	,['ABC', 'D']     )







   def      _UpperCAmelCase  (     self     )      -> Dict:


        '''simple docstring'''


        lowercase_      :							str						=      Trie()
        lowercase_      :							int						=      trie.cut_text('ABC'	,[0, 0, 2, 1, 2, 3]     )
        self.assertEqual(UpperCAmelCase_	,['AB', 'C']     )



"""simple docstring"""


import unittest

from transformers import GPTNeoXJapaneseConfig, is_torch_available
from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import GPTNeoXJapaneseTokenizer
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
      import torch

      from transformers import GPTNeoXJapaneseForCausalLM, GPTNeoXJapaneseModel



class 	__a	:






       def __init__(	self			:				Optional[int]							,   UpperCAmelCase_			:				Any							,   UpperCAmelCase_			:				str=13							,   UpperCAmelCase_			:				Optional[int]=7							,   UpperCAmelCase_			:				Optional[int]=True							,   UpperCAmelCase_			:				Optional[Any]=True							,   UpperCAmelCase_			:				Optional[int]=True							,   UpperCAmelCase_			:				int=True							,   UpperCAmelCase_			:				Dict=99							,   UpperCAmelCase_			:				str=32							,   UpperCAmelCase_			:				List[Any]=5							,   UpperCAmelCase_			:				str=4							,   UpperCAmelCase_			:				Union[str, Any]=4							,   UpperCAmelCase_			:				Optional[int]="gelu"							,   UpperCAmelCase_			:				Tuple=0.0							,   UpperCAmelCase_			:				Any=0.1							,   UpperCAmelCase_			:				int=True							,   UpperCAmelCase_			:				Union[str, Any]=512							,   UpperCAmelCase_			:				int=16							,   UpperCAmelCase_			:				Optional[Any]=2							,   UpperCAmelCase_			:				str=0.02							,   UpperCAmelCase_			:				Any=3							,   UpperCAmelCase_			:				Any=4							,   UpperCAmelCase_			:				Optional[Any]=None							,   )->	Tuple:







              """simple docstring"""
              UpperCamelCase       			=		parent
              UpperCamelCase       			=		batch_size
              UpperCamelCase       			=		seq_length
              UpperCamelCase       			=		is_training
              UpperCamelCase       			=		use_input_mask
              UpperCamelCase       			=		use_token_type_ids
              UpperCamelCase       			=		use_labels
              UpperCamelCase       			=		vocab_size
              UpperCamelCase       			=		hidden_size
              UpperCamelCase       			=		num_hidden_layers
              UpperCamelCase       			=		num_attention_heads
              UpperCamelCase       			=		intermediate_multiple_size
              UpperCamelCase       			=		hidden_act
              UpperCamelCase       			=		hidden_dropout
              UpperCamelCase       			=		attention_dropout
              UpperCamelCase       			=		weight_tying
              UpperCamelCase       			=		max_position_embeddings
              UpperCamelCase       			=		type_vocab_size
              UpperCamelCase       			=		type_sequence_label_size
              UpperCamelCase       			=		initializer_range
              UpperCamelCase       			=		num_labels
              UpperCamelCase       			=		num_choices
              UpperCamelCase       			=		scope






       def 						_SCREAMING_SNAKE_CASE							(	self			:				Optional[int]      )->	Union[str, Any]:







              """simple docstring"""
              UpperCamelCase       			=		ids_tensor([self.batch_size, self.seq_length]							,   self.vocab_size      )

              UpperCamelCase       			=		None
              if self.use_input_mask:
                     UpperCamelCase       			=		random_attention_mask([self.batch_size, self.seq_length]      )

              UpperCamelCase       			=		None
              if self.use_labels:
                     UpperCamelCase       			=		ids_tensor([self.batch_size, self.seq_length]							,   self.num_labels      )

              UpperCamelCase       			=		self.get_config()

              return config, input_ids, input_mask, token_labels






       def 						_SCREAMING_SNAKE_CASE							(	self			:				Dict      )->	Dict:







              """simple docstring"""
              return GPTNeoXJapaneseConfig(
                  vocab_size=self.vocab_size							,   hidden_size=self.hidden_size							,   num_hidden_layers=self.num_hidden_layers							,   num_attention_heads=self.num_attention_heads							,   intermediate_multiple_size=self.intermediate_multiple_size							,   hidden_act=self.hidden_act							,   hidden_dropout=self.hidden_dropout							,   attention_dropout=self.attention_dropout							,   weight_tying=self.weight_tying							,   max_position_embeddings=self.max_position_embeddings							,   type_vocab_size=self.type_vocab_size							,   is_decoder=UpperCAmelCase_							,   initializer_range=self.initializer_range							,   )






       def 						_SCREAMING_SNAKE_CASE							(	self			:				Any      )->	List[Any]:







              """simple docstring"""
              UpperCamelCase			,							UpperCamelCase			,							UpperCamelCase			,							UpperCamelCase       			=		self.prepare_config_and_inputs()

              UpperCamelCase       			=		True

              return config, input_ids, input_mask, token_labels






       def 						_SCREAMING_SNAKE_CASE							(	self			:				Union[str, Any]							,   UpperCAmelCase_			:				List[Any]							,   UpperCAmelCase_			:				Union[str, Any]							,   UpperCAmelCase_			:				List[Any]      )->	List[str]:







              """simple docstring"""
              UpperCamelCase       			=		GPTNeoXJapaneseModel(config=UpperCAmelCase_      )
              model.to(UpperCAmelCase_      )
              model.eval()
              UpperCamelCase       			=		model(UpperCAmelCase_							,   attention_mask=UpperCAmelCase_      )
              UpperCamelCase       			=		model(UpperCAmelCase_      )
              self.parent.assertEqual(result.last_hidden_state.shape							,   (self.batch_size, self.seq_length, self.hidden_size)      )






       def 						_SCREAMING_SNAKE_CASE							(	self			:				Any							,   UpperCAmelCase_			:				List[str]							,   UpperCAmelCase_			:				List[Any]							,   UpperCAmelCase_			:				List[str]      )->	List[str]:







              """simple docstring"""
              UpperCamelCase       			=		True
              UpperCamelCase       			=		GPTNeoXJapaneseModel(UpperCAmelCase_      )
              model.to(UpperCAmelCase_      )
              model.eval()
              UpperCamelCase       			=		model(UpperCAmelCase_							,   attention_mask=UpperCAmelCase_      )
              self.parent.assertEqual(result.last_hidden_state.shape							,   (self.batch_size, self.seq_length, self.hidden_size)      )






       def 						_SCREAMING_SNAKE_CASE							(	self			:				List[str]							,   UpperCAmelCase_			:				Tuple							,   UpperCAmelCase_			:				Dict							,   UpperCAmelCase_			:				List[Any]							,   UpperCAmelCase_			:				Optional[Any]      )->	Any:







              """simple docstring"""
              UpperCamelCase       			=		GPTNeoXJapaneseForCausalLM(config=UpperCAmelCase_      )
              model.to(UpperCAmelCase_      )
              model.eval()
              UpperCamelCase       			=		model(UpperCAmelCase_							,   attention_mask=UpperCAmelCase_							,   labels=UpperCAmelCase_      )
              self.parent.assertEqual(result.logits.shape							,   (self.batch_size, self.seq_length, self.vocab_size)      )






       def 						_SCREAMING_SNAKE_CASE							(	self			:				Dict							,   UpperCAmelCase_			:				int							,   UpperCAmelCase_			:				Optional[int]							,   UpperCAmelCase_			:				Optional[Any]      )->	Any:







              """simple docstring"""
              UpperCamelCase       			=		True
              UpperCamelCase       			=		GPTNeoXJapaneseForCausalLM(config=UpperCAmelCase_      )
              model.to(UpperCAmelCase_      )
              model.eval()

              # first forward pass
              UpperCamelCase       			=		model(UpperCAmelCase_							,   attention_mask=UpperCAmelCase_							,   use_cache=UpperCAmelCase_      )
              UpperCamelCase       			=		outputs.past_key_values

              # create hypothetical multiple next token and extent to next_input_ids
              UpperCamelCase       			=		ids_tensor((self.batch_size, 3)							,   config.vocab_size      )
              UpperCamelCase       			=		ids_tensor((self.batch_size, 3)							,   vocab_size=2      )

              # append to next input_ids and
              UpperCamelCase       			=		torch.cat([input_ids, next_tokens]							,   dim=-1      )
              UpperCamelCase       			=		torch.cat([input_mask, next_mask]							,   dim=-1      )

              UpperCamelCase       			=		model(UpperCAmelCase_							,   attention_mask=UpperCAmelCase_							,   output_hidden_states=UpperCAmelCase_      )
              UpperCamelCase       			=		output_from_no_past["hidden_states"][0]
              UpperCamelCase       			=		model(
                  UpperCAmelCase_							,   attention_mask=UpperCAmelCase_							,   past_key_values=UpperCAmelCase_							,   output_hidden_states=UpperCAmelCase_							,   )["hidden_states"][0]

              # select random slice
              UpperCamelCase       			=		ids_tensor((1,)							,   output_from_past.shape[-1]      ).item()
              UpperCamelCase       			=		output_from_no_past[:, -3:, random_slice_idx].detach()
              UpperCamelCase       			=		output_from_past[:, :, random_slice_idx].detach()

              self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]      )

              # test that outputs are equal for slice
              self.parent.assertTrue(torch.allclose(UpperCAmelCase_							,   UpperCAmelCase_							,   atol=1e-3      )      )






       def 						_SCREAMING_SNAKE_CASE							(	self			:				Optional[Any]      )->	Any:







              """simple docstring"""
              UpperCamelCase       			=		self.prepare_config_and_inputs()
              UpperCamelCase			,							UpperCamelCase			,							UpperCamelCase			,							UpperCamelCase       			=		config_and_inputs
              UpperCamelCase       			=		{"input_ids": input_ids, "attention_mask": input_mask}
              return config, inputs_dict





@require_torch
class 	__a	(   _lowerCAmelCase     ,  _lowerCAmelCase     ,  unittest.TestCase						):
       UpperCamelCase_		:       Union[str, Any]				=      (GPTNeoXJapaneseModel, GPTNeoXJapaneseForCausalLM) if is_torch_available() else ()
       UpperCamelCase_		:       Optional[int]				=      (GPTNeoXJapaneseForCausalLM,) if is_torch_available() else ()
       UpperCamelCase_		:       List[str]				=      (
           {'''feature-extraction''': GPTNeoXJapaneseModel, '''text-generation''': GPTNeoXJapaneseForCausalLM}
           if is_torch_available()
           else {}
       )
       UpperCamelCase_		:       Optional[Any]				=      False
       UpperCamelCase_		:       Tuple				=      False
       UpperCamelCase_		:       Optional[int]				=      False
       UpperCamelCase_		:       List[Any]				=      False






       def 						_SCREAMING_SNAKE_CASE							(	self			:				List[str]      )->	List[str]:







              """simple docstring"""
              UpperCamelCase       			=		GPTNeoXJapaneseModelTester(self      )
              UpperCamelCase       			=		ConfigTester(self							,   config_class=UpperCAmelCase_							,   hidden_size=37      )






       def 						_SCREAMING_SNAKE_CASE							(	self			:				str      )->	Any:







              """simple docstring"""
              self.config_tester.run_common_tests()






       def 						_SCREAMING_SNAKE_CASE							(	self			:				Optional[int]      )->	Dict:







              """simple docstring"""
              UpperCamelCase			,							UpperCamelCase			,							UpperCamelCase			,							UpperCamelCase       			=		self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_model(UpperCAmelCase_							,   UpperCAmelCase_							,   UpperCAmelCase_      )






       def 						_SCREAMING_SNAKE_CASE							(	self			:				List[Any]      )->	Optional[Any]:







              """simple docstring"""
              UpperCamelCase			,							UpperCamelCase			,							UpperCamelCase			,							UpperCamelCase       			=		self.model_tester.prepare_config_and_inputs_for_decoder()
              self.model_tester.create_and_check_model_as_decoder(UpperCAmelCase_							,   UpperCAmelCase_							,   UpperCAmelCase_      )






       def 						_SCREAMING_SNAKE_CASE							(	self			:				Dict      )->	List[Any]:







              """simple docstring"""
              # This regression test was failing with PyTorch < 1.3
              UpperCamelCase			,							UpperCamelCase			,							UpperCamelCase			,							UpperCamelCase       			=		self.model_tester.prepare_config_and_inputs_for_decoder()

              UpperCamelCase       			=		None

              self.model_tester.create_and_check_model_as_decoder(UpperCAmelCase_							,   UpperCAmelCase_							,   UpperCAmelCase_      )






       def 						_SCREAMING_SNAKE_CASE							(	self			:				List[Any]      )->	List[Any]:







              """simple docstring"""
              UpperCamelCase			,							UpperCamelCase			,							UpperCamelCase			,							UpperCamelCase       			=		self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_decoder_model_past_large_inputs(UpperCAmelCase_							,   UpperCAmelCase_							,   UpperCAmelCase_      )






       def 						_SCREAMING_SNAKE_CASE							(	self			:				List[Any]      )->	Optional[int]:







              """simple docstring"""
              UpperCamelCase       			=		self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_for_causal_lm(*UpperCAmelCase_      )






       @slow
       def 						_SCREAMING_SNAKE_CASE							(	self			:				Dict      )->	str:







              """simple docstring"""
              UpperCamelCase       			=		"abeja/gpt-neox-japanese-2.7b"

              UpperCamelCase       			=		["データサイエンティストとは、", "100年後に必要とされる会社は、", "フルリモートの環境で働くために必要なことは、", "国境の長いトンネルを抜けると", "美味しい日本食といえば、"]

              UpperCamelCase       			=		[
                  "データサイエンティストとは、データを分析し、ビジネスに役立つ知見を導き出す専門家のことです。",
                  "100年後に必要とされる会社は、「人」が中心の会社です。",
                  "フルリモートの環境で働くために必要なことは、「自分の時間をコントロールする」ことです。",
                  "国境の長いトンネルを抜けると、そこは雪国だった。",
                  "美味しい日本食といえば、やっぱりお寿司ですよね。",
              ]

              UpperCamelCase       			=		GPTNeoXJapaneseTokenizer.from_pretrained(UpperCAmelCase_      )
              UpperCamelCase       			=		GPTNeoXJapaneseForCausalLM.from_pretrained(UpperCAmelCase_      )

              UpperCamelCase       			=		[]
              for prompt in prompts:
                     UpperCamelCase       			=		tokenizer(UpperCAmelCase_							,   return_tensors="pt"      ).input_ids
                     UpperCamelCase       			=		model.generate(UpperCAmelCase_							,   max_length=50      )
                     UpperCamelCase       			=		tokenizer.batch_decode(UpperCAmelCase_							,   skip_special_tokens=UpperCAmelCase_      )
                     predicted_outputs += generated_string
              self.assertListEqual(UpperCAmelCase_							,   UpperCAmelCase_      )





