


'''simple docstring'''



import shutil
import tempfile
import unittest

from transformers import (
    SPIECE_UNDERLINE,
    AddedToken,
    BatchEncoding,
    NllbTokenizer,
    NllbTokenizerFast,
    is_torch_available,
)
from transformers.testing_utils import (
    get_tests_dir,
    nested_simplify,
    require_sentencepiece,
    require_tokenizers,
    require_torch,
)

from ...test_tokenization_common import TokenizerTesterMixin


A__						:   Union[str, Any]				=    get_tests_dir("""fixtures/test_sentencepiece.model""")


if is_torch_available():
      from transformers.models.mam_aaa.modeling_mam_aaa import shift_tokens_right

A__						:   str				=    256047
A__						:   int				=    256145





@require_sentencepiece
@require_tokenizers
class   UpperCAmelCase_    (_UpperCAmelCase   ,						unittest.TestCase    ):





    """simple docstring"""






    lowerCamelCase    :   List[Any]					=     NllbTokenizer
    lowerCamelCase    :   Optional[int]					=     NllbTokenizerFast
    lowerCamelCase    :   List[Any]					=     True
    lowerCamelCase    :   Dict					=     True
    lowerCamelCase    :   int					=     {}







    def  lowercase_							(				self      )							->	str:
         super().setUp()

         # We have a SentencePiece fixture for testing
         __lowerCamelCase  :      Union[str, Any]														=						NllbTokenizer(SCREAMING_SNAKE_CASE_     ,	keep_accents=SCREAMING_SNAKE_CASE_      )
         tokenizer.save_pretrained(self.tmpdirname      )







    def  lowercase_							(				self      )							->	Tuple:
         __lowerCamelCase  :      str														=						NllbTokenizer(SCREAMING_SNAKE_CASE_     ,	keep_accents=SCREAMING_SNAKE_CASE_      )

         __lowerCamelCase  :      str														=						tokenizer.tokenize('This is a test'      )
         self.assertListEqual(SCREAMING_SNAKE_CASE_     ,	['▁This', '▁is', '▁a', '▁t', 'est']      )

         self.assertListEqual(
             tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_      )     ,	[value + tokenizer.fairseq_offset for value in [2_85, 46, 10, 1_70, 3_82]]     ,	)

         __lowerCamelCase  :      Dict														=						tokenizer.tokenize('I was born in 92000, and this is falsé.'      )
         self.assertListEqual(
             SCREAMING_SNAKE_CASE_     ,	[
                 SPIECE_UNDERLINE + 'I',
                 SPIECE_UNDERLINE + 'was',
                 SPIECE_UNDERLINE + 'b',
                 'or',
                 'n',
                 SPIECE_UNDERLINE + 'in',
                 SPIECE_UNDERLINE + '',
                 '9',
                 '2',
                 '0',
                 '0',
                 '0',
                 ',',
                 SPIECE_UNDERLINE + 'and',
                 SPIECE_UNDERLINE + 'this',
                 SPIECE_UNDERLINE + 'is',
                 SPIECE_UNDERLINE + 'f',
                 'al',
                 's',
                 'é',
                 '.',
             ]     ,	)
         __lowerCamelCase  :      int														=						tokenizer.convert_tokens_to_ids(SCREAMING_SNAKE_CASE_      )
         self.assertListEqual(
             SCREAMING_SNAKE_CASE_     ,	[
                 value + tokenizer.fairseq_offset
                 for value in [8, 21, 84, 55, 24, 19, 7, 2, 6_02, 3_47, 3_47, 3_47, 3, 12, 66, 46, 72, 80, 6, 2, 4]
             ]     ,	)

         __lowerCamelCase  :      Any														=						tokenizer.convert_ids_to_tokens(SCREAMING_SNAKE_CASE_      )
         self.assertListEqual(
             SCREAMING_SNAKE_CASE_     ,	[
                 SPIECE_UNDERLINE + 'I',
                 SPIECE_UNDERLINE + 'was',
                 SPIECE_UNDERLINE + 'b',
                 'or',
                 'n',
                 SPIECE_UNDERLINE + 'in',
                 SPIECE_UNDERLINE + '',
                 '<unk>',
                 '2',
                 '0',
                 '0',
                 '0',
                 ',',
                 SPIECE_UNDERLINE + 'and',
                 SPIECE_UNDERLINE + 'this',
                 SPIECE_UNDERLINE + 'is',
                 SPIECE_UNDERLINE + 'f',
                 'al',
                 's',
                 '<unk>',
                 '.',
             ]     ,	)







    def  lowercase_							(				self      )							->	Tuple:
         __lowerCamelCase  :      Union[str, Any]														=						(self.rust_tokenizer_class, 'hf-internal-testing/tiny-random-nllb', {})
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'      ):
                   __lowerCamelCase  :      Optional[Any]														=						self.rust_tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_     ,	**SCREAMING_SNAKE_CASE_      )
                   __lowerCamelCase  :      Tuple														=						self.tokenizer_class.from_pretrained(SCREAMING_SNAKE_CASE_     ,	**SCREAMING_SNAKE_CASE_      )

                   __lowerCamelCase  :      Optional[int]														=						tempfile.mkdtemp()

                   __lowerCamelCase  :      Any														=						tokenizer_r.save_pretrained(SCREAMING_SNAKE_CASE_      )
                   __lowerCamelCase  :      Any														=						tokenizer_p.save_pretrained(SCREAMING_SNAKE_CASE_      )

                   # Checks it save with the same files + the tokenizer.json file for the fast one
                   self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files      )      )
                   __lowerCamelCase  :      List[str]														=						tuple(f for f in tokenizer_r_files if 'tokenizer.json' not in f      )
                   self.assertSequenceEqual(SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_      )

                   # Checks everything loads correctly in the same way
                   __lowerCamelCase  :      Tuple														=						tokenizer_r.from_pretrained(SCREAMING_SNAKE_CASE_      )
                   __lowerCamelCase  :      int														=						tokenizer_p.from_pretrained(SCREAMING_SNAKE_CASE_      )

                   # Check special tokens are set accordingly on Rust and Python
                   for key in tokenizer_pp.special_tokens_map:
                        self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_      )      )

                   shutil.rmtree(SCREAMING_SNAKE_CASE_      )

                   # Save tokenizer rust, legacy_format=True
                   __lowerCamelCase  :      Any														=						tempfile.mkdtemp()

                   __lowerCamelCase  :      int														=						tokenizer_r.save_pretrained(SCREAMING_SNAKE_CASE_     ,	legacy_format=SCREAMING_SNAKE_CASE_      )
                   __lowerCamelCase  :      Any														=						tokenizer_p.save_pretrained(SCREAMING_SNAKE_CASE_      )

                   # Checks it save with the same files
                   self.assertSequenceEqual(SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_      )

                   # Checks everything loads correctly in the same way
                   __lowerCamelCase  :      Optional[Any]														=						tokenizer_r.from_pretrained(SCREAMING_SNAKE_CASE_      )
                   __lowerCamelCase  :      Any														=						tokenizer_p.from_pretrained(SCREAMING_SNAKE_CASE_      )

                   # Check special tokens are set accordingly on Rust and Python
                   for key in tokenizer_pp.special_tokens_map:
                        self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_      )      )

                   shutil.rmtree(SCREAMING_SNAKE_CASE_      )

                   # Save tokenizer rust, legacy_format=False
                   __lowerCamelCase  :      Any														=						tempfile.mkdtemp()

                   __lowerCamelCase  :      int														=						tokenizer_r.save_pretrained(SCREAMING_SNAKE_CASE_     ,	legacy_format=SCREAMING_SNAKE_CASE_      )
                   __lowerCamelCase  :      Optional[int]														=						tokenizer_p.save_pretrained(SCREAMING_SNAKE_CASE_      )

                   # Checks it saved the tokenizer.json file
                   self.assertTrue(any('tokenizer.json' in f for f in tokenizer_r_files      )      )

                   # Checks everything loads correctly in the same way
                   __lowerCamelCase  :      str														=						tokenizer_r.from_pretrained(SCREAMING_SNAKE_CASE_      )
                   __lowerCamelCase  :      Tuple														=						tokenizer_p.from_pretrained(SCREAMING_SNAKE_CASE_      )

                   # Check special tokens are set accordingly on Rust and Python
                   for key in tokenizer_pp.special_tokens_map:
                        self.assertTrue(hasattr(SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_      )      )

                   shutil.rmtree(SCREAMING_SNAKE_CASE_      )







    @require_torch
    def  lowercase_							(				self      )							->	Optional[int]:
         if not self.test_seqaseq:
              return

         __lowerCamelCase  :      Optional[int]														=						self.get_tokenizers()
         for tokenizer in tokenizers:
              with self.subTest(f'{tokenizer.__class__.__name__}'      ):
                   # Longer text that will definitely require truncation.
                   __lowerCamelCase  :      Union[str, Any]														=						[
                       ' UN Chief Says There Is No Military Solution in Syria',
                       ' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for'
                       ' Syria is that \'there is no military solution\' to the nearly five-year conflict and more weapons'
                       ' will only worsen the violence and misery for millions of people.',
                   ]
                   __lowerCamelCase  :      List[str]														=						[
                       'Şeful ONU declară că nu există o soluţie militară în Siria',
                       'Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al'
                       ' Rusiei pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi'
                       ' că noi arme nu vor face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.',
                   ]
                   try:
                        __lowerCamelCase  :      Optional[Any]														=						tokenizer.prepare_seqaseq_batch(
                            src_texts=SCREAMING_SNAKE_CASE_     ,	tgt_texts=SCREAMING_SNAKE_CASE_     ,	max_length=3     ,	max_target_length=10     ,	return_tensors='pt'     ,	src_lang='eng_Latn'     ,	tgt_lang='ron_Latn'     ,	)
                   except NotImplementedError:
                        return
                   self.assertEqual(batch.input_ids.shape[1]     ,	3      )
                   self.assertEqual(batch.labels.shape[1]     ,	10      )
                   # max_target_length will default to max_length if not specified
                   __lowerCamelCase  :      int														=						tokenizer.prepare_seqaseq_batch(
                       SCREAMING_SNAKE_CASE_     ,	tgt_texts=SCREAMING_SNAKE_CASE_     ,	max_length=3     ,	return_tensors='pt'      )
                   self.assertEqual(batch.input_ids.shape[1]     ,	3      )
                   self.assertEqual(batch.labels.shape[1]     ,	3      )

                   __lowerCamelCase  :      List[str]														=						tokenizer.prepare_seqaseq_batch(
                       src_texts=SCREAMING_SNAKE_CASE_     ,	max_length=3     ,	max_target_length=10     ,	return_tensors='pt'      )
                   self.assertEqual(batch_encoder_only.input_ids.shape[1]     ,	3      )
                   self.assertEqual(batch_encoder_only.attention_mask.shape[1]     ,	3      )
                   self.assertNotIn('decoder_input_ids'     ,	SCREAMING_SNAKE_CASE_      )







    @unittest.skip('Unfortunately way too slow to build a BPE with SentencePiece.'      )
    def  lowercase_							(				self      )							->	Optional[Any]:
         pass







    def  lowercase_							(				self      )							->	Tuple:
         for tokenizer, pretrained_name, kwargs in self.tokenizers_list:
              with self.subTest(f'{tokenizer.__class__.__name__} ({pretrained_name})'      ):
                   __lowerCamelCase  :      Dict														=						[AddedToken('<special>'     ,	lstrip=SCREAMING_SNAKE_CASE_      )]

                   __lowerCamelCase  :      Dict														=						self.rust_tokenizer_class.from_pretrained(
                       SCREAMING_SNAKE_CASE_     ,	additional_special_tokens=SCREAMING_SNAKE_CASE_     ,	**SCREAMING_SNAKE_CASE_      )
                   __lowerCamelCase  :      List[Any]														=						tokenizer_r.encode('Hey this is a <special> token'      )

                   __lowerCamelCase  :      Union[str, Any]														=						tokenizer_r.encode('<special>'     ,	add_special_tokens=SCREAMING_SNAKE_CASE_      )[0]

                   self.assertTrue(special_token_id in r_output      )

                   if self.test_slow_tokenizer:
                        __lowerCamelCase  :      Optional[int]														=						self.rust_tokenizer_class.from_pretrained(
                            SCREAMING_SNAKE_CASE_     ,	additional_special_tokens=SCREAMING_SNAKE_CASE_     ,	**SCREAMING_SNAKE_CASE_     ,	)
                        __lowerCamelCase  :      Dict														=						self.tokenizer_class.from_pretrained(
                            SCREAMING_SNAKE_CASE_     ,	additional_special_tokens=SCREAMING_SNAKE_CASE_     ,	**SCREAMING_SNAKE_CASE_      )

                        __lowerCamelCase  :      str														=						tokenizer_p.encode('Hey this is a <special> token'      )

                        __lowerCamelCase  :      Union[str, Any]														=						tokenizer_cr.encode('Hey this is a <special> token'      )

                        self.assertEqual(SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_      )
                        self.assertEqual(SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_      )
                        self.assertTrue(special_token_id in p_output      )
                        self.assertTrue(special_token_id in cr_output      )







@require_torch
@require_sentencepiece
@require_tokenizers
class   UpperCAmelCase_    (unittest.TestCase    ):





    """simple docstring"""






    lowerCamelCase    :   Optional[int]					=     'facebook/nllb-200-distilled-600M'
    lowerCamelCase    :   List[str]					=     [
        ' UN Chief Says There Is No Military Solution in Syria',
        ' Secretary-General Ban Ki-moon says his response to Russia\'s stepped up military support for Syria is that "there is no military solution" to the nearly five-year conflict and more weapons will only worsen the violence and misery for millions of people.',
    ]
    lowerCamelCase    :   List[str]					=     [
        'Şeful ONU declară că nu există o soluţie militară în Siria',
        'Secretarul General Ban Ki-moon declară că răspunsul său la intensificarea sprijinului militar al Rusiei'
        ' pentru Siria este că "nu există o soluţie militară" la conflictul de aproape cinci ani şi că noi arme nu vor'
        ' face decât să înrăutăţească violenţele şi mizeria pentru milioane de oameni.',
    ]
    lowerCamelCase    :   Tuple					=     [
        2_5_6_0_4_7,
        1_6_2_9_7,
        1_3_4_4_0_8,
        8_1_6_5,
        2_4_8_0_6_6,
        1_4_7_3_4,
        9_5_0,
        1_1_3_5,
        1_0_5_7_2_1,
        3_5_7_3,
        8_3,
        2_7_3_5_2,
        1_0_8,
        4_9_4_8_6,
        2,
    ]







    @classmethod
    def  lowercase_							(				cls      )							->	Union[str, Any]:
         __lowerCamelCase  :      NllbTokenizer														=						NllbTokenizer.from_pretrained(
             cls.checkpoint_name     ,	src_lang='eng_Latn'     ,	tgt_lang='ron_Latn'      )
         __lowerCamelCase  :      Dict														=						1
         return cls







    def  lowercase_							(				self      )							->	List[Any]:
         self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ace_Arab']     ,	25_60_01      )
         self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['ace_Latn']     ,	25_60_02      )
         self.assertEqual(self.tokenizer.fairseq_tokens_to_ids['fra_Latn']     ,	25_60_57      )







    def  lowercase_							(				self      )							->	Any:
         __lowerCamelCase  :      Optional[Any]														=						self.tokenizer.batch_encode_plus(self.src_text      ).input_ids[0]
         self.assertListEqual(self.expected_src_tokens     ,	SCREAMING_SNAKE_CASE_      )







    def  lowercase_							(				self      )							->	List[Any]:
         self.assertIn(SCREAMING_SNAKE_CASE_     ,	self.tokenizer.all_special_ids      )
         # fmt: off
         __lowerCamelCase  :      List[Any]														=						[RO_CODE, 42_54, 9_80_68, 11_29_23, 3_90_72, 39_09, 7_13, 10_27_67, 26, 1_73_14, 3_56_42, 1_46_83, 3_31_18, 20_22, 6_69_87, 2, 25_60_47]
         # fmt: on

         __lowerCamelCase  :      Optional[int]														=						self.tokenizer.decode(SCREAMING_SNAKE_CASE_     ,	skip_special_tokens=SCREAMING_SNAKE_CASE_      )
         __lowerCamelCase  :      Union[str, Any]														=						self.tokenizer.decode(generated_ids[1:]     ,	skip_special_tokens=SCREAMING_SNAKE_CASE_      )
         self.assertEqual(SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_      )
         self.assertNotIn(self.tokenizer.eos_token     ,	SCREAMING_SNAKE_CASE_      )







    def  lowercase_							(				self      )							->	int:
         __lowerCamelCase  :      Tuple														=						['this is gunna be a long sentence ' * 20]
         assert isinstance(src_text[0]     ,	SCREAMING_SNAKE_CASE_      )
         __lowerCamelCase  :      str														=						10
         __lowerCamelCase  :      List[Any]														=						self.tokenizer(SCREAMING_SNAKE_CASE_     ,	max_length=SCREAMING_SNAKE_CASE_     ,	truncation=SCREAMING_SNAKE_CASE_      ).input_ids[0]
         self.assertEqual(ids[-1]     ,	2      )
         self.assertEqual(ids[0]     ,	SCREAMING_SNAKE_CASE_      )
         self.assertEqual(len(SCREAMING_SNAKE_CASE_      )     ,	SCREAMING_SNAKE_CASE_      )







    def  lowercase_							(				self      )							->	Union[str, Any]:
         self.assertListEqual(self.tokenizer.convert_tokens_to_ids(['<mask>', 'ar_AR']      )     ,	[25_62_03, 3]      )







    def  lowercase_							(				self      )							->	str:
         __lowerCamelCase  :      Dict														=						tempfile.mkdtemp()
         __lowerCamelCase  :      List[Any]														=						self.tokenizer.fairseq_tokens_to_ids
         self.tokenizer.save_pretrained(SCREAMING_SNAKE_CASE_      )
         __lowerCamelCase  :      Any														=						NllbTokenizer.from_pretrained(SCREAMING_SNAKE_CASE_      )
         self.assertDictEqual(new_tok.fairseq_tokens_to_ids     ,	SCREAMING_SNAKE_CASE_      )







    @require_torch
    def  lowercase_							(				self      )							->	Any:
         __lowerCamelCase  :      Tuple														=						self.tokenizer(
             self.src_text     ,	text_target=self.tgt_text     ,	padding=SCREAMING_SNAKE_CASE_     ,	truncation=SCREAMING_SNAKE_CASE_     ,	max_length=len(self.expected_src_tokens      )     ,	return_tensors='pt'     ,	)
         __lowerCamelCase  :      Dict														=						shift_tokens_right(
             batch['labels']     ,	self.tokenizer.pad_token_id     ,	self.tokenizer.lang_code_to_id['ron_Latn']      )

         self.assertIsInstance(SCREAMING_SNAKE_CASE_     ,	SCREAMING_SNAKE_CASE_      )

         self.assertEqual((2, 15)     ,	batch.input_ids.shape      )
         self.assertEqual((2, 15)     ,	batch.attention_mask.shape      )
         __lowerCamelCase  :      List[Any]														=						batch.input_ids.tolist()[0]
         self.assertListEqual(self.expected_src_tokens     ,	SCREAMING_SNAKE_CASE_      )
         self.assertEqual(SCREAMING_SNAKE_CASE_     ,	batch.decoder_input_ids[0, 0]      )  # EOS
         # Test that special tokens are reset
         self.assertEqual(self.tokenizer.prefix_tokens     ,	[EN_CODE]      )
         self.assertEqual(self.tokenizer.suffix_tokens     ,	[self.tokenizer.eos_token_id]      )







    def  lowercase_							(				self      )							->	Dict:
         __lowerCamelCase  :      int														=						self.tokenizer(self.src_text     ,	padding=SCREAMING_SNAKE_CASE_     ,	truncation=SCREAMING_SNAKE_CASE_     ,	max_length=3     ,	return_tensors='pt'      )
         __lowerCamelCase  :      Tuple														=						self.tokenizer(
             text_target=self.tgt_text     ,	padding=SCREAMING_SNAKE_CASE_     ,	truncation=SCREAMING_SNAKE_CASE_     ,	max_length=10     ,	return_tensors='pt'      )
         __lowerCamelCase  :      Optional[int]														=						targets['input_ids']
         __lowerCamelCase  :      List[Any]														=						shift_tokens_right(
             SCREAMING_SNAKE_CASE_     ,	self.tokenizer.pad_token_id     ,	decoder_start_token_id=self.tokenizer.lang_code_to_id[self.tokenizer.tgt_lang]     ,	)

         self.assertEqual(batch.input_ids.shape[1]     ,	3      )
         self.assertEqual(batch.decoder_input_ids.shape[1]     ,	10      )







    @require_torch
    def  lowercase_							(				self      )							->	Optional[int]:
         __lowerCamelCase  :      Tuple														=						self.tokenizer._build_translation_inputs(
             'A test'     ,	return_tensors='pt'     ,	src_lang='eng_Latn'     ,	tgt_lang='fra_Latn'      )

         self.assertEqual(
             nested_simplify(SCREAMING_SNAKE_CASE_      )     ,	{
                 # A, test, EOS, en_XX
                 'input_ids': [[25_60_47, 70, 73_56, 2]],
                 'attention_mask': [[1, 1, 1, 1]],
                 # ar_AR
                 'forced_bos_token_id': 25_60_57,
             }     ,	)







    @require_torch
    def  lowercase_							(				self      )							->	List[Any]:
         __lowerCamelCase  :      List[Any]														=						True
         __lowerCamelCase  :      Any														=						self.tokenizer(
             'UN Chief says there is no military solution in Syria'     ,	src_lang='eng_Latn'     ,	tgt_lang='fra_Latn'      )
         self.assertEqual(
             inputs.input_ids     ,	[1_62_97, 13_44_08, 2_56_53, 63_70, 2_48, 2_54, 10_39_29, 9_49_95, 1_08, 4_94_86, 2, 25_60_47]      )

         __lowerCamelCase  :      List[Any]														=						False
         __lowerCamelCase  :      Union[str, Any]														=						self.tokenizer(
             'UN Chief says there is no military solution in Syria'     ,	src_lang='eng_Latn'     ,	tgt_lang='fra_Latn'      )
         self.assertEqual(
             inputs.input_ids     ,	[25_60_47, 1_62_97, 13_44_08, 2_56_53, 63_70, 2_48, 2_54, 10_39_29, 9_49_95, 1_08, 4_94_86, 2]      )








"""simple docstring"""


import unittest

from transformers import GPTNeoXJapaneseConfig, is_torch_available
from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import GPTNeoXJapaneseTokenizer
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
      import torch

      from transformers import GPTNeoXJapaneseForCausalLM, GPTNeoXJapaneseModel



class 	__a	:






       def __init__(	self			:				Optional[int]							,   UpperCAmelCase_			:				Any							,   UpperCAmelCase_			:				str=13							,   UpperCAmelCase_			:				Optional[int]=7							,   UpperCAmelCase_			:				Optional[int]=True							,   UpperCAmelCase_			:				Optional[Any]=True							,   UpperCAmelCase_			:				Optional[int]=True							,   UpperCAmelCase_			:				int=True							,   UpperCAmelCase_			:				Dict=99							,   UpperCAmelCase_			:				str=32							,   UpperCAmelCase_			:				List[Any]=5							,   UpperCAmelCase_			:				str=4							,   UpperCAmelCase_			:				Union[str, Any]=4							,   UpperCAmelCase_			:				Optional[int]="gelu"							,   UpperCAmelCase_			:				Tuple=0.0							,   UpperCAmelCase_			:				Any=0.1							,   UpperCAmelCase_			:				int=True							,   UpperCAmelCase_			:				Union[str, Any]=512							,   UpperCAmelCase_			:				int=16							,   UpperCAmelCase_			:				Optional[Any]=2							,   UpperCAmelCase_			:				str=0.02							,   UpperCAmelCase_			:				Any=3							,   UpperCAmelCase_			:				Any=4							,   UpperCAmelCase_			:				Optional[Any]=None							,   )->	Tuple:







              """simple docstring"""
              UpperCamelCase       			=		parent
              UpperCamelCase       			=		batch_size
              UpperCamelCase       			=		seq_length
              UpperCamelCase       			=		is_training
              UpperCamelCase       			=		use_input_mask
              UpperCamelCase       			=		use_token_type_ids
              UpperCamelCase       			=		use_labels
              UpperCamelCase       			=		vocab_size
              UpperCamelCase       			=		hidden_size
              UpperCamelCase       			=		num_hidden_layers
              UpperCamelCase       			=		num_attention_heads
              UpperCamelCase       			=		intermediate_multiple_size
              UpperCamelCase       			=		hidden_act
              UpperCamelCase       			=		hidden_dropout
              UpperCamelCase       			=		attention_dropout
              UpperCamelCase       			=		weight_tying
              UpperCamelCase       			=		max_position_embeddings
              UpperCamelCase       			=		type_vocab_size
              UpperCamelCase       			=		type_sequence_label_size
              UpperCamelCase       			=		initializer_range
              UpperCamelCase       			=		num_labels
              UpperCamelCase       			=		num_choices
              UpperCamelCase       			=		scope






       def 						_SCREAMING_SNAKE_CASE							(	self			:				Optional[int]      )->	Union[str, Any]:







              """simple docstring"""
              UpperCamelCase       			=		ids_tensor([self.batch_size, self.seq_length]							,   self.vocab_size      )

              UpperCamelCase       			=		None
              if self.use_input_mask:
                     UpperCamelCase       			=		random_attention_mask([self.batch_size, self.seq_length]      )

              UpperCamelCase       			=		None
              if self.use_labels:
                     UpperCamelCase       			=		ids_tensor([self.batch_size, self.seq_length]							,   self.num_labels      )

              UpperCamelCase       			=		self.get_config()

              return config, input_ids, input_mask, token_labels






       def 						_SCREAMING_SNAKE_CASE							(	self			:				Dict      )->	Dict:







              """simple docstring"""
              return GPTNeoXJapaneseConfig(
                  vocab_size=self.vocab_size							,   hidden_size=self.hidden_size							,   num_hidden_layers=self.num_hidden_layers							,   num_attention_heads=self.num_attention_heads							,   intermediate_multiple_size=self.intermediate_multiple_size							,   hidden_act=self.hidden_act							,   hidden_dropout=self.hidden_dropout							,   attention_dropout=self.attention_dropout							,   weight_tying=self.weight_tying							,   max_position_embeddings=self.max_position_embeddings							,   type_vocab_size=self.type_vocab_size							,   is_decoder=UpperCAmelCase_							,   initializer_range=self.initializer_range							,   )






       def 						_SCREAMING_SNAKE_CASE							(	self			:				Any      )->	List[Any]:







              """simple docstring"""
              UpperCamelCase			,							UpperCamelCase			,							UpperCamelCase			,							UpperCamelCase       			=		self.prepare_config_and_inputs()

              UpperCamelCase       			=		True

              return config, input_ids, input_mask, token_labels






       def 						_SCREAMING_SNAKE_CASE							(	self			:				Union[str, Any]							,   UpperCAmelCase_			:				List[Any]							,   UpperCAmelCase_			:				Union[str, Any]							,   UpperCAmelCase_			:				List[Any]      )->	List[str]:







              """simple docstring"""
              UpperCamelCase       			=		GPTNeoXJapaneseModel(config=UpperCAmelCase_      )
              model.to(UpperCAmelCase_      )
              model.eval()
              UpperCamelCase       			=		model(UpperCAmelCase_							,   attention_mask=UpperCAmelCase_      )
              UpperCamelCase       			=		model(UpperCAmelCase_      )
              self.parent.assertEqual(result.last_hidden_state.shape							,   (self.batch_size, self.seq_length, self.hidden_size)      )






       def 						_SCREAMING_SNAKE_CASE							(	self			:				Any							,   UpperCAmelCase_			:				List[str]							,   UpperCAmelCase_			:				List[Any]							,   UpperCAmelCase_			:				List[str]      )->	List[str]:







              """simple docstring"""
              UpperCamelCase       			=		True
              UpperCamelCase       			=		GPTNeoXJapaneseModel(UpperCAmelCase_      )
              model.to(UpperCAmelCase_      )
              model.eval()
              UpperCamelCase       			=		model(UpperCAmelCase_							,   attention_mask=UpperCAmelCase_      )
              self.parent.assertEqual(result.last_hidden_state.shape							,   (self.batch_size, self.seq_length, self.hidden_size)      )






       def 						_SCREAMING_SNAKE_CASE							(	self			:				List[str]							,   UpperCAmelCase_			:				Tuple							,   UpperCAmelCase_			:				Dict							,   UpperCAmelCase_			:				List[Any]							,   UpperCAmelCase_			:				Optional[Any]      )->	Any:







              """simple docstring"""
              UpperCamelCase       			=		GPTNeoXJapaneseForCausalLM(config=UpperCAmelCase_      )
              model.to(UpperCAmelCase_      )
              model.eval()
              UpperCamelCase       			=		model(UpperCAmelCase_							,   attention_mask=UpperCAmelCase_							,   labels=UpperCAmelCase_      )
              self.parent.assertEqual(result.logits.shape							,   (self.batch_size, self.seq_length, self.vocab_size)      )






       def 						_SCREAMING_SNAKE_CASE							(	self			:				Dict							,   UpperCAmelCase_			:				int							,   UpperCAmelCase_			:				Optional[int]							,   UpperCAmelCase_			:				Optional[Any]      )->	Any:







              """simple docstring"""
              UpperCamelCase       			=		True
              UpperCamelCase       			=		GPTNeoXJapaneseForCausalLM(config=UpperCAmelCase_      )
              model.to(UpperCAmelCase_      )
              model.eval()

              # first forward pass
              UpperCamelCase       			=		model(UpperCAmelCase_							,   attention_mask=UpperCAmelCase_							,   use_cache=UpperCAmelCase_      )
              UpperCamelCase       			=		outputs.past_key_values

              # create hypothetical multiple next token and extent to next_input_ids
              UpperCamelCase       			=		ids_tensor((self.batch_size, 3)							,   config.vocab_size      )
              UpperCamelCase       			=		ids_tensor((self.batch_size, 3)							,   vocab_size=2      )

              # append to next input_ids and
              UpperCamelCase       			=		torch.cat([input_ids, next_tokens]							,   dim=-1      )
              UpperCamelCase       			=		torch.cat([input_mask, next_mask]							,   dim=-1      )

              UpperCamelCase       			=		model(UpperCAmelCase_							,   attention_mask=UpperCAmelCase_							,   output_hidden_states=UpperCAmelCase_      )
              UpperCamelCase       			=		output_from_no_past["hidden_states"][0]
              UpperCamelCase       			=		model(
                  UpperCAmelCase_							,   attention_mask=UpperCAmelCase_							,   past_key_values=UpperCAmelCase_							,   output_hidden_states=UpperCAmelCase_							,   )["hidden_states"][0]

              # select random slice
              UpperCamelCase       			=		ids_tensor((1,)							,   output_from_past.shape[-1]      ).item()
              UpperCamelCase       			=		output_from_no_past[:, -3:, random_slice_idx].detach()
              UpperCamelCase       			=		output_from_past[:, :, random_slice_idx].detach()

              self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]      )

              # test that outputs are equal for slice
              self.parent.assertTrue(torch.allclose(UpperCAmelCase_							,   UpperCAmelCase_							,   atol=1e-3      )      )






       def 						_SCREAMING_SNAKE_CASE							(	self			:				Optional[Any]      )->	Any:







              """simple docstring"""
              UpperCamelCase       			=		self.prepare_config_and_inputs()
              UpperCamelCase			,							UpperCamelCase			,							UpperCamelCase			,							UpperCamelCase       			=		config_and_inputs
              UpperCamelCase       			=		{"input_ids": input_ids, "attention_mask": input_mask}
              return config, inputs_dict





@require_torch
class 	__a	(   _lowerCAmelCase     ,  _lowerCAmelCase     ,  unittest.TestCase						):
       UpperCamelCase_		:       Union[str, Any]				=      (GPTNeoXJapaneseModel, GPTNeoXJapaneseForCausalLM) if is_torch_available() else ()
       UpperCamelCase_		:       Optional[int]				=      (GPTNeoXJapaneseForCausalLM,) if is_torch_available() else ()
       UpperCamelCase_		:       List[str]				=      (
           {'''feature-extraction''': GPTNeoXJapaneseModel, '''text-generation''': GPTNeoXJapaneseForCausalLM}
           if is_torch_available()
           else {}
       )
       UpperCamelCase_		:       Optional[Any]				=      False
       UpperCamelCase_		:       Tuple				=      False
       UpperCamelCase_		:       Optional[int]				=      False
       UpperCamelCase_		:       List[Any]				=      False






       def 						_SCREAMING_SNAKE_CASE							(	self			:				List[str]      )->	List[str]:







              """simple docstring"""
              UpperCamelCase       			=		GPTNeoXJapaneseModelTester(self      )
              UpperCamelCase       			=		ConfigTester(self							,   config_class=UpperCAmelCase_							,   hidden_size=37      )






       def 						_SCREAMING_SNAKE_CASE							(	self			:				str      )->	Any:







              """simple docstring"""
              self.config_tester.run_common_tests()






       def 						_SCREAMING_SNAKE_CASE							(	self			:				Optional[int]      )->	Dict:







              """simple docstring"""
              UpperCamelCase			,							UpperCamelCase			,							UpperCamelCase			,							UpperCamelCase       			=		self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_model(UpperCAmelCase_							,   UpperCAmelCase_							,   UpperCAmelCase_      )






       def 						_SCREAMING_SNAKE_CASE							(	self			:				List[Any]      )->	Optional[Any]:







              """simple docstring"""
              UpperCamelCase			,							UpperCamelCase			,							UpperCamelCase			,							UpperCamelCase       			=		self.model_tester.prepare_config_and_inputs_for_decoder()
              self.model_tester.create_and_check_model_as_decoder(UpperCAmelCase_							,   UpperCAmelCase_							,   UpperCAmelCase_      )






       def 						_SCREAMING_SNAKE_CASE							(	self			:				Dict      )->	List[Any]:







              """simple docstring"""
              # This regression test was failing with PyTorch < 1.3
              UpperCamelCase			,							UpperCamelCase			,							UpperCamelCase			,							UpperCamelCase       			=		self.model_tester.prepare_config_and_inputs_for_decoder()

              UpperCamelCase       			=		None

              self.model_tester.create_and_check_model_as_decoder(UpperCAmelCase_							,   UpperCAmelCase_							,   UpperCAmelCase_      )






       def 						_SCREAMING_SNAKE_CASE							(	self			:				List[Any]      )->	List[Any]:







              """simple docstring"""
              UpperCamelCase			,							UpperCamelCase			,							UpperCamelCase			,							UpperCamelCase       			=		self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_decoder_model_past_large_inputs(UpperCAmelCase_							,   UpperCAmelCase_							,   UpperCAmelCase_      )






       def 						_SCREAMING_SNAKE_CASE							(	self			:				List[Any]      )->	Optional[int]:







              """simple docstring"""
              UpperCamelCase       			=		self.model_tester.prepare_config_and_inputs()
              self.model_tester.create_and_check_for_causal_lm(*UpperCAmelCase_      )






       @slow
       def 						_SCREAMING_SNAKE_CASE							(	self			:				Dict      )->	str:







              """simple docstring"""
              UpperCamelCase       			=		"abeja/gpt-neox-japanese-2.7b"

              UpperCamelCase       			=		["データサイエンティストとは、", "100年後に必要とされる会社は、", "フルリモートの環境で働くために必要なことは、", "国境の長いトンネルを抜けると", "美味しい日本食といえば、"]

              UpperCamelCase       			=		[
                  "データサイエンティストとは、データを分析し、ビジネスに役立つ知見を導き出す専門家のことです。",
                  "100年後に必要とされる会社は、「人」が中心の会社です。",
                  "フルリモートの環境で働くために必要なことは、「自分の時間をコントロールする」ことです。",
                  "国境の長いトンネルを抜けると、そこは雪国だった。",
                  "美味しい日本食といえば、やっぱりお寿司ですよね。",
              ]

              UpperCamelCase       			=		GPTNeoXJapaneseTokenizer.from_pretrained(UpperCAmelCase_      )
              UpperCamelCase       			=		GPTNeoXJapaneseForCausalLM.from_pretrained(UpperCAmelCase_      )

              UpperCamelCase       			=		[]
              for prompt in prompts:
                     UpperCamelCase       			=		tokenizer(UpperCAmelCase_							,   return_tensors="pt"      ).input_ids
                     UpperCamelCase       			=		model.generate(UpperCAmelCase_							,   max_length=50      )
                     UpperCamelCase       			=		tokenizer.batch_decode(UpperCAmelCase_							,   skip_special_tokens=UpperCAmelCase_      )
                     predicted_outputs += generated_string
              self.assertListEqual(UpperCAmelCase_							,   UpperCAmelCase_      )





