




"""simple docstring"""
# Note: if you intend to run this script make sure you look under scripts/fsmt/
# to locate the appropriate script to do the work correctly. There is a set of scripts to:
# - download and prepare data and run the conversion script
# - perform eval to get the best hparam into the config
# - generate model_cards - useful if you have multiple models from the same paper

import argparse
import json
import os
import re
from collections import OrderedDict
from os.path import basename, dirname

import fairseq
import torch
from fairseq import hub_utils
from fairseq.data.dictionary import Dictionary

from transformers import FSMTConfig, FSMTForConditionalGeneration
from transformers.models.fsmt.tokenization_fsmt import VOCAB_FILES_NAMES
from transformers.tokenization_utils_base import TOKENIZER_CONFIG_FILE
from transformers.utils import WEIGHTS_NAME, logging


logging.set_verbosity_warning()

a_				:    Optional[Any]										=     2

# based on the results of a search on a range of `num_beams`, `length_penalty` and `early_stopping`
# values against wmt19 test data to obtain the best BLEU scores, we will use the following defaults:
#
# * `num_beams`: 5 (higher scores better, but requires more memory/is slower, can be adjusted by users)
# * `early_stopping`: `False` consistently scored better
# * `length_penalty` varied, so will assign the best one depending on the model
a_				:    Optional[int]										=     {
    # fairseq:
    '''wmt19-ru-en''': {'''length_penalty''': 1.1},
    '''wmt19-en-ru''': {'''length_penalty''': 1.1_5},
    '''wmt19-en-de''': {'''length_penalty''': 1.0},
    '''wmt19-de-en''': {'''length_penalty''': 1.1},
    # allenai:
    '''wmt16-en-de-dist-12-1''': {'''length_penalty''': 0.6},
    '''wmt16-en-de-dist-6-1''': {'''length_penalty''': 0.6},
    '''wmt16-en-de-12-1''': {'''length_penalty''': 0.8},
    '''wmt19-de-en-6-6-base''': {'''length_penalty''': 0.6},
    '''wmt19-de-en-6-6-big''': {'''length_penalty''': 0.6},
}

# this remaps the different models to their organization names
a_				:    Optional[int]										=     {}
for m in ["wmt19-ru-en", "wmt19-en-ru", "wmt19-en-de", "wmt19-de-en"]:
					a_				:    Dict										=     '''facebook'''
for m in [
    "wmt16-en-de-dist-12-1",
    "wmt16-en-de-dist-6-1",
    "wmt16-en-de-12-1",
    "wmt19-de-en-6-6-base",
    "wmt19-de-en-6-6-big",
]:
					a_				:    Tuple										=     '''allenai'''





def 						UpperCAmelCase				(			A__:    List[str]     )  ->     int:
			# (1) remove word breaking symbol, (2) add word ending symbol where the word is not broken up,
			# e.g.: d = {'le@@': 5, 'tt@@': 6, 'er': 7} => {'le': 5, 'tt': 6, 'er</w>': 7}
			__lowerCamelCase					:	Any	       =				dict((re.sub(r'@@$'							,				''							,				A__     ), v) if k.endswith('@@'     ) else (re.sub(r'$'							,				'</w>'							,				A__     ), v) for k, v in d.items()     )
			__lowerCamelCase					:	Any	       =				'<s> <pad> </s> <unk>'.split()
			# restore the special tokens
			for k in keep_keys:
						del da[f'''{k}</w>''']
						__lowerCamelCase					:	Tuple	       =				d[k]  # restore
			return da





def 						UpperCAmelCase				(			A__:    str							,				A__:    Any     )  ->     Tuple:
			# prep
			assert os.path.exists(A__     )
			os.makedirs(A__							,				exist_ok=A__     )
			print(f'''Writing results to {pytorch_dump_folder_path}'''     )

			# handle various types of models

			__lowerCamelCase					:	str	       =				basename(A__     )
			__lowerCamelCase					:	Tuple	       =				dirname(A__     )

			__lowerCamelCase					:	Optional[int]	       =				fairseq.model_parallel.models.transformer.ModelParallelTransformerModel
			__lowerCamelCase					:	str	       =				cls.hub_models()
			__lowerCamelCase					:	Union[str, Any]	       =				{'bpe': 'fastbpe', 'tokenizer': 'moses'}
			__lowerCamelCase					:	List[Any]	       =				'.'
			# note: since the model dump is old, fairseq has upgraded its model some
			# time later, and it does a whole lot of rewrites and splits on the saved
			# weights, therefore we can't use torch.load() directly on the model file.
			# see: upgrade_state_dict(state_dict) in fairseq_model.py
			print(f'''using checkpoint {checkpoint_file}'''     )
			__lowerCamelCase					:	Any	       =				hub_utils.from_pretrained(
			    A__							,				A__							,				A__							,				archive_map=A__							,				**A__     )

			__lowerCamelCase					:	Tuple	       =				vars(chkpt['args']['model']     )

			__lowerCamelCase					:	Union[str, Any]	       =				args['source_lang']
			__lowerCamelCase					:	Optional[Any]	       =				args['target_lang']

			__lowerCamelCase					:	List[Any]	       =				dirname(A__     )
			__lowerCamelCase					:	List[str]	       =				basename(A__     )

			# dicts
			__lowerCamelCase					:	Union[str, Any]	       =				os.path.join(A__							,				f'''dict.{src_lang}.txt'''     )
			__lowerCamelCase					:	List[str]	       =				os.path.join(A__							,				f'''dict.{tgt_lang}.txt'''     )

			__lowerCamelCase					:	Optional[int]	       =				Dictionary.load(A__     )
			__lowerCamelCase					:	int	       =				rewrite_dict_keys(src_dict.indices     )
			__lowerCamelCase					:	Optional[int]	       =				len(A__     )
			__lowerCamelCase					:	List[Any]	       =				os.path.join(A__							,				'vocab-src.json'     )
			print(f'''Generating {src_vocab_file} of {src_vocab_size} of {src_lang} records'''     )
			with open(A__							,				'w'							,				encoding='utf-8'     ) as f:
						f.write(json.dumps(A__							,				ensure_ascii=A__							,				indent=A__     )     )

			# detect whether this is a do_lower_case situation, which can be derived by checking whether we
			# have at least one uppercase letter in the source vocab
			__lowerCamelCase					:	Dict	       =				True
			for k in src_vocab.keys():
						if not k.islower():
									__lowerCamelCase					:	Tuple	       =				False
									break

			__lowerCamelCase					:	List[str]	       =				Dictionary.load(A__     )
			__lowerCamelCase					:	Tuple	       =				rewrite_dict_keys(tgt_dict.indices     )
			__lowerCamelCase					:	Tuple	       =				len(A__     )
			__lowerCamelCase					:	Union[str, Any]	       =				os.path.join(A__							,				'vocab-tgt.json'     )
			print(f'''Generating {tgt_vocab_file} of {tgt_vocab_size} of {tgt_lang} records'''     )
			with open(A__							,				'w'							,				encoding='utf-8'     ) as f:
						f.write(json.dumps(A__							,				ensure_ascii=A__							,				indent=A__     )     )

			# merges_file (bpecodes)
			__lowerCamelCase					:	Optional[int]	       =				os.path.join(A__							,				VOCAB_FILES_NAMES['merges_file']     )
			for fn in ["bpecodes", "code"]:  # older fairseq called the merges file "code"
						__lowerCamelCase					:	List[Any]	       =				os.path.join(A__							,				A__     )
						if os.path.exists(A__     ):
									break
			with open(A__							,				encoding='utf-8'     ) as fin:
						__lowerCamelCase					:	int	       =				fin.read()
			__lowerCamelCase					:	int	       =				re.sub(r' \d+$'							,				''							,				A__							,				0							,				re.M     )  # remove frequency number
			print(f'''Generating {merges_file}'''     )
			with open(A__							,				'w'							,				encoding='utf-8'     ) as fout:
						fout.write(A__     )

			# model config
			__lowerCamelCase					:	List[str]	       =				os.path.join(A__							,				'config.json'     )

			# validate bpe/tokenizer config, as currently it's hardcoded to moses+fastbpe -
			# may have to modify the tokenizer if a different type is used by a future model
			assert args["bpe"] == "fastbpe", f'''need to extend tokenizer to support bpe={args["bpe"]}'''
			assert args["tokenizer"] == "moses", f'''need to extend tokenizer to support bpe={args["tokenizer"]}'''

			__lowerCamelCase					:	Union[str, Any]	       =				{
			    'architectures': ['FSMTForConditionalGeneration'],
			    'model_type': 'fsmt',
			    'activation_dropout': args['activation_dropout'],
			    'activation_function': 'relu',
			    'attention_dropout': args['attention_dropout'],
			    'd_model': args['decoder_embed_dim'],
			    'dropout': args['dropout'],
			    'init_std': 0.02,
			    'max_position_embeddings': args['max_source_positions'],
			    'num_hidden_layers': args['encoder_layers'],
			    'src_vocab_size': src_vocab_size,
			    'tgt_vocab_size': tgt_vocab_size,
			    'langs': [src_lang, tgt_lang],
			    'encoder_attention_heads': args['encoder_attention_heads'],
			    'encoder_ffn_dim': args['encoder_ffn_embed_dim'],
			    'encoder_layerdrop': args['encoder_layerdrop'],
			    'encoder_layers': args['encoder_layers'],
			    'decoder_attention_heads': args['decoder_attention_heads'],
			    'decoder_ffn_dim': args['decoder_ffn_embed_dim'],
			    'decoder_layerdrop': args['decoder_layerdrop'],
			    'decoder_layers': args['decoder_layers'],
			    'bos_token_id': 0,
			    'pad_token_id': 1,
			    'eos_token_id': 2,
			    'is_encoder_decoder': True,
			    'scale_embedding': not args['no_scale_embedding'],
			    'tie_word_embeddings': args['share_all_embeddings'],
			}

			# good hparam defaults to start with
			__lowerCamelCase					:	List[Any]	       =				5
			__lowerCamelCase					:	int	       =				False
			if model_dir in best_score_hparams and "length_penalty" in best_score_hparams[model_dir]:
						__lowerCamelCase					:	Optional[Any]	       =				best_score_hparams[model_dir]['length_penalty']
			else:
						__lowerCamelCase					:	Optional[int]	       =				1.0

			print(f'''Generating {fsmt_model_config_file}'''     )
			with open(A__							,				'w'							,				encoding='utf-8'     ) as f:
						f.write(json.dumps(A__							,				ensure_ascii=A__							,				indent=A__     )     )

			# tokenizer config
			__lowerCamelCase					:	Dict	       =				os.path.join(A__							,				A__     )

			__lowerCamelCase					:	int	       =				{
			    'langs': [src_lang, tgt_lang],
			    'model_max_length': 1024,
			    'do_lower_case': do_lower_case,
			}

			print(f'''Generating {fsmt_tokenizer_config_file}'''     )
			with open(A__							,				'w'							,				encoding='utf-8'     ) as f:
						f.write(json.dumps(A__							,				ensure_ascii=A__							,				indent=A__     )     )

			# model
			__lowerCamelCase					:	Optional[Any]	       =				chkpt['models'][0]
			__lowerCamelCase					:	Any	       =				model.state_dict()

			# rename keys to start with 'model.'
			__lowerCamelCase					:	Dict	       =				OrderedDict(('model.' + k, v) for k, v in model_state_dict.items()     )

			# remove unneeded keys
			__lowerCamelCase					:	Dict	       =				[
			    'model.model',
			    'model.encoder.version',
			    'model.decoder.version',
			    'model.encoder_embed_tokens.weight',
			    'model.decoder_embed_tokens.weight',
			    'model.encoder.embed_positions._float_tensor',
			    'model.decoder.embed_positions._float_tensor',
			]
			for k in ignore_keys:
						model_state_dict.pop(A__							,				A__     )

			__lowerCamelCase					:	Any	       =				FSMTConfig.from_pretrained(A__     )
			__lowerCamelCase					:	Optional[int]	       =				FSMTForConditionalGeneration(A__     )

			# check that it loads ok
			model_new.load_state_dict(A__							,				strict=A__     )

			# save
			__lowerCamelCase					:	Optional[int]	       =				os.path.join(A__							,				A__     )
			print(f'''Generating {pytorch_weights_dump_path}'''     )
			torch.save(A__							,				A__     )

			print('Conversion is done!'     )
			print('\nLast step is to upload the files to s3'     )
			print(f'''cd {data_root}'''     )
			print(f'''transformers-cli upload {model_dir}'''     )


if __name__ == "__main__":
					a_				:    Dict										=     argparse.ArgumentParser()
					# Required parameters
					parser.add_argument(
					    '''--fsmt_checkpoint_path''',
					    default=None,
					    type=str,
					    required=True,
					    help=(
					        '''Path to the official PyTorch checkpoint file which is expected to reside in the dump dir with dicts,'''
					        ''' bpecodes, etc.'''
					    ),
					)
					parser.add_argument(
					    '''--pytorch_dump_folder_path''', default=None, type=str, required=True, help='''Path to the output PyTorch model.'''
					)
					a_				:    Union[str, Any]										=     parser.parse_args()
					convert_fsmt_checkpoint_to_pytorch(args.fsmt_checkpoint_path, args.pytorch_dump_folder_path)









"""simple docstring"""
import unittest

from transformers import GPTNeoXJapaneseConfig, is_torch_available
from transformers.models.gpt_neox_japanese.tokenization_gpt_neox_japanese import GPTNeoXJapaneseTokenizer
from transformers.testing_utils import require_torch, slow, torch_device

from ...test_configuration_common import ConfigTester
from ...test_modeling_common import ModelTesterMixin, ids_tensor, random_attention_mask
from ...test_pipeline_mixin import PipelineTesterMixin


if is_torch_available():
					import torch

					from transformers import GPTNeoXJapaneseForCausalLM, GPTNeoXJapaneseModel

class __lowercase:




					'''simple docstring'''


					def __init__(    self						,	__a						,	__a=13						,	__a=7						,	__a=True						,	__a=True						,	__a=True						,	__a=True						,	__a=99						,	__a=32						,	__a=5						,	__a=4						,	__a=4						,	__a="gelu"						,	__a=0.0						,	__a=0.1						,	__a=True						,	__a=512						,	__a=16						,	__a=2						,	__a=0.02						,	__a=3						,	__a=4						,	__a=None						,	):
								__lowerCamelCase					:	List[str]	       =				parent
								__lowerCamelCase					:	List[str]	       =				batch_size
								__lowerCamelCase					:	Optional[Any]	       =				seq_length
								__lowerCamelCase					:	List[Any]	       =				is_training
								__lowerCamelCase					:	str	       =				use_input_mask
								__lowerCamelCase					:	List[str]	       =				use_token_type_ids
								__lowerCamelCase					:	str	       =				use_labels
								__lowerCamelCase					:	Tuple	       =				vocab_size
								__lowerCamelCase					:	Any	       =				hidden_size
								__lowerCamelCase					:	List[str]	       =				num_hidden_layers
								__lowerCamelCase					:	Tuple	       =				num_attention_heads
								__lowerCamelCase					:	str	       =				intermediate_multiple_size
								__lowerCamelCase					:	Tuple	       =				hidden_act
								__lowerCamelCase					:	str	       =				hidden_dropout
								__lowerCamelCase					:	List[str]	       =				attention_dropout
								__lowerCamelCase					:	Tuple	       =				weight_tying
								__lowerCamelCase					:	int	       =				max_position_embeddings
								__lowerCamelCase					:	Tuple	       =				type_vocab_size
								__lowerCamelCase					:	Union[str, Any]	       =				type_sequence_label_size
								__lowerCamelCase					:	Optional[int]	       =				initializer_range
								__lowerCamelCase					:	Optional[int]	       =				num_labels
								__lowerCamelCase					:	str	       =				num_choices
								__lowerCamelCase					:	Tuple	       =				scope
					def     snake_case_				(    self     ):
								__lowerCamelCase					:	List[Any]	       =				ids_tensor([self.batch_size, self.seq_length]						,	self.vocab_size     )

								__lowerCamelCase					:	str	       =				None
								if self.use_input_mask:
											__lowerCamelCase					:	List[str]	       =				random_attention_mask([self.batch_size, self.seq_length]     )

								__lowerCamelCase					:	Optional[Any]	       =				None
								if self.use_labels:
											__lowerCamelCase					:	List[str]	       =				ids_tensor([self.batch_size, self.seq_length]						,	self.num_labels     )

								__lowerCamelCase					:	int	       =				self.get_config()

								return config, input_ids, input_mask, token_labels
					def     snake_case_				(    self     ):
								return GPTNeoXJapaneseConfig(
								    vocab_size=self.vocab_size						,	hidden_size=self.hidden_size						,	num_hidden_layers=self.num_hidden_layers						,	num_attention_heads=self.num_attention_heads						,	intermediate_multiple_size=self.intermediate_multiple_size						,	hidden_act=self.hidden_act						,	hidden_dropout=self.hidden_dropout						,	attention_dropout=self.attention_dropout						,	weight_tying=self.weight_tying						,	max_position_embeddings=self.max_position_embeddings						,	type_vocab_size=self.type_vocab_size						,	is_decoder=__a						,	initializer_range=self.initializer_range						,	)
					def     snake_case_				(    self     ):
								__lowerCamelCase       ,      __lowerCamelCase       ,      __lowerCamelCase       ,      __lowerCamelCase					:	Union[str, Any]	       =				self.prepare_config_and_inputs()

								__lowerCamelCase					:	Tuple	       =				True

								return config, input_ids, input_mask, token_labels
					def     snake_case_				(    self						,	__a						,	__a						,	__a     ):
								__lowerCamelCase					:	Any	       =				GPTNeoXJapaneseModel(config=__a     )
								model.to(__a     )
								model.eval()
								__lowerCamelCase					:	int	       =				model(__a						,	attention_mask=__a     )
								__lowerCamelCase					:	Optional[Any]	       =				model(__a     )
								self.parent.assertEqual(result.last_hidden_state.shape						,	(self.batch_size, self.seq_length, self.hidden_size)     )
					def     snake_case_				(    self						,	__a						,	__a						,	__a     ):
								__lowerCamelCase					:	List[str]	       =				True
								__lowerCamelCase					:	Tuple	       =				GPTNeoXJapaneseModel(__a     )
								model.to(__a     )
								model.eval()
								__lowerCamelCase					:	Optional[Any]	       =				model(__a						,	attention_mask=__a     )
								self.parent.assertEqual(result.last_hidden_state.shape						,	(self.batch_size, self.seq_length, self.hidden_size)     )
					def     snake_case_				(    self						,	__a						,	__a						,	__a						,	__a     ):
								__lowerCamelCase					:	Optional[Any]	       =				GPTNeoXJapaneseForCausalLM(config=__a     )
								model.to(__a     )
								model.eval()
								__lowerCamelCase					:	List[str]	       =				model(__a						,	attention_mask=__a						,	labels=__a     )
								self.parent.assertEqual(result.logits.shape						,	(self.batch_size, self.seq_length, self.vocab_size)     )
					def     snake_case_				(    self						,	__a						,	__a						,	__a     ):
								__lowerCamelCase					:	List[Any]	       =				True
								__lowerCamelCase					:	str	       =				GPTNeoXJapaneseForCausalLM(config=__a     )
								model.to(__a     )
								model.eval()

								# first forward pass
								__lowerCamelCase					:	List[Any]	       =				model(__a						,	attention_mask=__a						,	use_cache=__a     )
								__lowerCamelCase					:	Any	       =				outputs.past_key_values

								# create hypothetical multiple next token and extent to next_input_ids
								__lowerCamelCase					:	int	       =				ids_tensor((self.batch_size, 3)						,	config.vocab_size     )
								__lowerCamelCase					:	Tuple	       =				ids_tensor((self.batch_size, 3)						,	vocab_size=2     )

								# append to next input_ids and
								__lowerCamelCase					:	Any	       =				torch.cat([input_ids, next_tokens]						,	dim=-1     )
								__lowerCamelCase					:	int	       =				torch.cat([input_mask, next_mask]						,	dim=-1     )

								__lowerCamelCase					:	int	       =				model(__a						,	attention_mask=__a						,	output_hidden_states=__a     )
								__lowerCamelCase					:	Dict	       =				output_from_no_past['hidden_states'][0]
								__lowerCamelCase					:	Any	       =				model(
								    __a						,	attention_mask=__a						,	past_key_values=__a						,	output_hidden_states=__a						,	)['hidden_states'][0]

								# select random slice
								__lowerCamelCase					:	Any	       =				ids_tensor((1,)						,	output_from_past.shape[-1]     ).item()
								__lowerCamelCase					:	Dict	       =				output_from_no_past[:, -3:, random_slice_idx].detach()
								__lowerCamelCase					:	Any	       =				output_from_past[:, :, random_slice_idx].detach()

								self.parent.assertTrue(output_from_past_slice.shape[1] == next_tokens.shape[1]     )

								# test that outputs are equal for slice
								self.parent.assertTrue(torch.allclose(__a						,	__a						,	atol=1E-3     )     )





					def     snake_case_				(    self     ):
								__lowerCamelCase					:	List[str]	       =				self.prepare_config_and_inputs()
								__lowerCamelCase       ,      __lowerCamelCase       ,      __lowerCamelCase       ,      __lowerCamelCase					:	Any	       =				config_and_inputs
								__lowerCamelCase					:	Tuple	       =				{'input_ids': input_ids, 'attention_mask': input_mask}
								return config, inputs_dict



@require_torch
class __lowercase(      lowercase__	,	lowercase__	,	unittest.TestCase				):




					'''simple docstring'''


					__a						:						Dict     		= (GPTNeoXJapaneseModel, GPTNeoXJapaneseForCausalLM) if is_torch_available() else ()
					__a						:						str     		= (GPTNeoXJapaneseForCausalLM,) if is_torch_available() else ()
					__a						:						Optional[int]     		= (
					    {'feature-extraction': GPTNeoXJapaneseModel, 'text-generation': GPTNeoXJapaneseForCausalLM}
					    if is_torch_available()
					    else {}
					)
					__a						:						Union[str, Any]     		= False
					__a						:						Optional[Any]     		= False
					__a						:						List[Any]     		= False
					__a						:						List[Any]     		= False
					def     snake_case_				(    self     ):
								__lowerCamelCase					:	List[Any]	       =				GPTNeoXJapaneseModelTester(self     )
								__lowerCamelCase					:	List[Any]	       =				ConfigTester(self						,	config_class=__a						,	hidden_size=37     )
					def     snake_case_				(    self     ):
								self.config_tester.run_common_tests()
					def     snake_case_				(    self     ):
								__lowerCamelCase       ,      __lowerCamelCase       ,      __lowerCamelCase       ,      __lowerCamelCase					:	Union[str, Any]	       =				self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_model(__a						,	__a						,	__a     )
					def     snake_case_				(    self     ):
								__lowerCamelCase       ,      __lowerCamelCase       ,      __lowerCamelCase       ,      __lowerCamelCase					:	Tuple	       =				self.model_tester.prepare_config_and_inputs_for_decoder()
								self.model_tester.create_and_check_model_as_decoder(__a						,	__a						,	__a     )
					def     snake_case_				(    self     ):
								# This regression test was failing with PyTorch < 1.3
								__lowerCamelCase       ,      __lowerCamelCase       ,      __lowerCamelCase       ,      __lowerCamelCase					:	str	       =				self.model_tester.prepare_config_and_inputs_for_decoder()

								__lowerCamelCase					:	Tuple	       =				None

								self.model_tester.create_and_check_model_as_decoder(__a						,	__a						,	__a     )
					def     snake_case_				(    self     ):
								__lowerCamelCase       ,      __lowerCamelCase       ,      __lowerCamelCase       ,      __lowerCamelCase					:	Tuple	       =				self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_decoder_model_past_large_inputs(__a						,	__a						,	__a     )
					def     snake_case_				(    self     ):
								__lowerCamelCase					:	Optional[Any]	       =				self.model_tester.prepare_config_and_inputs()
								self.model_tester.create_and_check_for_causal_lm(*__a     )





					@slow
					def     snake_case_				(    self     ):
								__lowerCamelCase					:	List[Any]	       =				'abeja/gpt-neox-japanese-2.7b'

								__lowerCamelCase					:	Any	       =				['データサイエンティストとは、', '100年後に必要とされる会社は、', 'フルリモートの環境で働くために必要なことは、', '国境の長いトンネルを抜けると', '美味しい日本食といえば、']

								__lowerCamelCase					:	Union[str, Any]	       =				[
								    'データサイエンティストとは、データを分析し、ビジネスに役立つ知見を導き出す専門家のことです。',
								    '100年後に必要とされる会社は、「人」が中心の会社です。',
								    'フルリモートの環境で働くために必要なことは、「自分の時間をコントロールする」ことです。',
								    '国境の長いトンネルを抜けると、そこは雪国だった。',
								    '美味しい日本食といえば、やっぱりお寿司ですよね。',
								]

								__lowerCamelCase					:	Dict	       =				GPTNeoXJapaneseTokenizer.from_pretrained(__a     )
								__lowerCamelCase					:	str	       =				GPTNeoXJapaneseForCausalLM.from_pretrained(__a     )

								__lowerCamelCase					:	List[str]	       =				[]
								for prompt in prompts:
											__lowerCamelCase					:	List[Any]	       =				tokenizer(__a						,	return_tensors='pt'     ).input_ids
											__lowerCamelCase					:	List[Any]	       =				model.generate(__a						,	max_length=50     )
											__lowerCamelCase					:	str	       =				tokenizer.batch_decode(__a						,	skip_special_tokens=__a     )
											predicted_outputs += generated_string
								self.assertListEqual(__a						,	__a     )



