

import tempfile
import unittest

import numpy as np

import transformers
from transformers import GPTaTokenizer, GPTNeoConfig, is_flax_available, is_torch_available
from transformers.testing_utils import is_pt_flax_cross_test, require_flax, slow

from ...generation.test_flax_utils import FlaxGenerationTesterMixin
from ...test_modeling_flax_common import FlaxModelTesterMixin, ids_tensor, random_attention_mask


if is_flax_available():
					import jax
					import jax.numpy as jnp

					from transformers.modeling_flax_pytorch_utils import (
					    convert_pytorch_state_dict_to_flax,
					    load_flax_weights_in_pytorch_model,
					)
					from transformers.models.gpt_neo.modeling_flax_gpt_neo import FlaxGPTNeoForCausalLM, FlaxGPTNeoModel

if is_torch_available():
					import torch

class  a     :


						def __init__(				self  ,    A_  ,    A_=14  ,    A_=7  ,    A_=True  ,    A_=True  ,    A_=False  ,    A_=True  ,    A_=99  ,    A_=32  ,    A_=4  ,    A_=4  ,    A_=[[["global", "local"], 2]]  ,    A_=37  ,    A_="gelu"  ,    A_=0.1  ,    A_=0.1  ,    A_=512  ,    A_=7  ,    A_=0.02  ,    ):







										'''simple docstring'''


										_UpperCAmelCase				:						Optional[Any]       						=			parent
										_UpperCAmelCase				:						List[Any]       						=			batch_size
										_UpperCAmelCase				:						Optional[int]       						=			seq_length
										_UpperCAmelCase				:						Dict       						=			is_training
										_UpperCAmelCase				:						Tuple       						=			use_input_mask
										_UpperCAmelCase				:						Optional[Any]       						=			use_token_type_ids
										_UpperCAmelCase				:						Union[str, Any]       						=			use_labels
										_UpperCAmelCase				:						Dict       						=			vocab_size
										_UpperCAmelCase				:						Union[str, Any]       						=			hidden_size
										_UpperCAmelCase				:						Tuple       						=			num_hidden_layers
										_UpperCAmelCase				:						Tuple       						=			num_attention_heads
										_UpperCAmelCase				:						Optional[Any]       						=			attention_types
										_UpperCAmelCase				:						int       						=			intermediate_size
										_UpperCAmelCase				:						Union[str, Any]       						=			hidden_act
										_UpperCAmelCase				:						Tuple       						=			hidden_dropout_prob
										_UpperCAmelCase				:						Union[str, Any]       						=			attention_probs_dropout_prob
										_UpperCAmelCase				:						str       						=			max_position_embeddings
										_UpperCAmelCase				:						List[Any]       						=			window_size
										_UpperCAmelCase				:						Any       						=			initializer_range
										_UpperCAmelCase				:						List[str]       						=			None
										_UpperCAmelCase				:						Optional[Any]       						=			vocab_size - 1
										_UpperCAmelCase				:						Optional[int]       						=			vocab_size - 1
										_UpperCAmelCase				:						Dict       						=			vocab_size - 1


						def 				_UpperCAmelCase (				self		):







										'''simple docstring'''


										_UpperCAmelCase				:						List[Any]       						=			ids_tensor([self.batch_size, self.seq_length]  ,    self.vocab_size		)

										_UpperCAmelCase				:						Any       						=			None
										if self.use_input_mask:
														_UpperCAmelCase				:						Tuple       						=			random_attention_mask([self.batch_size, self.seq_length]		)

										_UpperCAmelCase				:						Optional[Any]       						=			GPTNeoConfig(
										    vocab_size=self.vocab_size  ,    hidden_size=self.hidden_size  ,    num_layers=self.num_hidden_layers  ,    num_heads=self.num_attention_heads  ,    max_position_embeddings=self.max_position_embeddings  ,    use_cache=lowerCamelCase__  ,    bos_token_id=self.bos_token_id  ,    eos_token_id=self.eos_token_id  ,    pad_token_id=self.pad_token_id  ,    window_size=self.window_size  ,    attention_types=self.attention_types  ,    )

										return (config, input_ids, input_mask)


						def 				_UpperCAmelCase (				self		):







										'''simple docstring'''


										_UpperCAmelCase				:						Any       						=			self.prepare_config_and_inputs()
										_UpperCAmelCase						,							_UpperCAmelCase						,							_UpperCAmelCase				:						Optional[Any]       						=			config_and_inputs
										_UpperCAmelCase				:						Dict       						=			{"input_ids": input_ids, "attention_mask": attention_mask}
										return config, inputs_dict


						def 				_UpperCAmelCase (				self  ,    A_  ,    A_  ,    A_  ,    A_		):







										'''simple docstring'''


										_UpperCAmelCase				:						List[Any]       						=			20
										_UpperCAmelCase				:						List[Any]       						=			model_class_name(lowerCamelCase__		)

										_UpperCAmelCase				:						Dict       						=			model.init_cache(input_ids.shape[0]  ,    lowerCamelCase__		)
										_UpperCAmelCase				:						int       						=			jnp.ones((input_ids.shape[0], max_decoder_length)  ,    dtype="i4"		)

										_UpperCAmelCase				:						Optional[int]       						=			jnp.broadcast_to(
										    jnp.arange(input_ids.shape[-1] - 1		)[None, :]  ,    (input_ids.shape[0], input_ids.shape[-1] - 1)		)
										_UpperCAmelCase				:						str       						=			model(
										    input_ids[:, :-1]  ,    attention_mask=lowerCamelCase__  ,    past_key_values=lowerCamelCase__  ,    position_ids=lowerCamelCase__  ,    )

										_UpperCAmelCase				:						List[str]       						=			jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]  ,    dtype="i4"		)
										_UpperCAmelCase				:						int       						=			model(
										    input_ids[:, -1:]  ,    attention_mask=lowerCamelCase__  ,    past_key_values=outputs_cache.past_key_values  ,    position_ids=lowerCamelCase__  ,    )

										_UpperCAmelCase				:						Dict       						=			model(lowerCamelCase__		)

										_UpperCAmelCase				:						Dict       						=			np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])		)		)
										self.parent.assertTrue(diff < 1e-3  ,    msg=f'Max diff is {diff}'		)





						def 				_UpperCAmelCase (				self  ,    A_  ,    A_  ,    A_  ,    A_		):







										'''simple docstring'''


										_UpperCAmelCase				:						Tuple       						=			20
										_UpperCAmelCase				:						Optional[Any]       						=			model_class_name(lowerCamelCase__		)

										_UpperCAmelCase				:						Any       						=			jnp.concatenate(
										    [attention_mask, jnp.zeros((attention_mask.shape[0], max_decoder_length - attention_mask.shape[1])		)]  ,    axis=-1  ,    )

										_UpperCAmelCase				:						List[str]       						=			model.init_cache(input_ids.shape[0]  ,    lowerCamelCase__		)
										_UpperCAmelCase				:						Union[str, Any]       						=			jnp.broadcast_to(
										    jnp.arange(input_ids.shape[-1] - 1		)[None, :]  ,    (input_ids.shape[0], input_ids.shape[-1] - 1)		)

										_UpperCAmelCase				:						List[str]       						=			model(
										    input_ids[:, :-1]  ,    attention_mask=lowerCamelCase__  ,    past_key_values=lowerCamelCase__  ,    position_ids=lowerCamelCase__  ,    )
										_UpperCAmelCase				:						Dict       						=			jnp.array(input_ids.shape[0] * [[input_ids.shape[-1] - 1]]  ,    dtype="i4"		)
										_UpperCAmelCase				:						List[Any]       						=			model(
										    input_ids[:, -1:]  ,    past_key_values=outputs_cache.past_key_values  ,    attention_mask=lowerCamelCase__  ,    position_ids=lowerCamelCase__  ,    )

										_UpperCAmelCase				:						Dict       						=			model(lowerCamelCase__  ,    attention_mask=lowerCamelCase__		)

										_UpperCAmelCase				:						List[Any]       						=			np.max(np.abs((outputs_cache_next[0][:, -1, :5] - outputs[0][:, -1, :5])		)		)
										self.parent.assertTrue(diff < 1e-3  ,    msg=f'Max diff is {diff}'		)







@require_flax
class  a     (		A__							,       A__							,       unittest.TestCase ):
						_lowercase			  =	(FlaxGPTNeoModel, FlaxGPTNeoForCausalLM) if is_flax_available() else ()
						_lowercase			  =	(FlaxGPTNeoForCausalLM,) if is_flax_available() else ()


						def 				_UpperCAmelCase (				self		):







										'''simple docstring'''


										_UpperCAmelCase				:						str       						=			FlaxGPTNeoModelTester(self		)


						def 				_UpperCAmelCase (				self		):







										'''simple docstring'''


										for model_class_name in self.all_model_classes:
														_UpperCAmelCase						,							_UpperCAmelCase						,							_UpperCAmelCase				:						Any       						=			self.model_tester.prepare_config_and_inputs()
														self.model_tester.check_use_cache_forward(lowerCamelCase__  ,    lowerCamelCase__  ,    lowerCamelCase__  ,    lowerCamelCase__		)


						def 				_UpperCAmelCase (				self		):







										'''simple docstring'''


										for model_class_name in self.all_model_classes:
														_UpperCAmelCase						,							_UpperCAmelCase						,							_UpperCAmelCase				:						Dict       						=			self.model_tester.prepare_config_and_inputs()
														self.model_tester.check_use_cache_forward_with_attn_mask(
														    lowerCamelCase__  ,    lowerCamelCase__  ,    lowerCamelCase__  ,    lowerCamelCase__		)


						@slow
						def 				_UpperCAmelCase (				self		):







										'''simple docstring'''


										_UpperCAmelCase				:						Tuple       						=			GPTaTokenizer.from_pretrained("gpt2"  ,    pad_token="<|endoftext|>"  ,    padding_side="left"		)
										_UpperCAmelCase				:						List[Any]       						=			tokenizer(["Hello this is a long string", "Hey"]  ,    return_tensors="np"  ,    padding=lowerCamelCase__  ,    truncation=lowerCamelCase__		)

										_UpperCAmelCase				:						Dict       						=			FlaxGPTNeoForCausalLM.from_pretrained("EleutherAI/gpt-neo-125M"		)
										_UpperCAmelCase				:						Optional[int]       						=			False
										_UpperCAmelCase				:						Optional[Any]       						=			model.config.eos_token_id

										_UpperCAmelCase				:						Any       						=			jax.jit(model.generate		)

										_UpperCAmelCase				:						Dict       						=			jit_generate(
										    inputs["input_ids"]  ,    attention_mask=inputs["attention_mask"]  ,    pad_token_id=tokenizer.pad_token_id		).sequences

										_UpperCAmelCase				:						Dict       						=			tokenizer.batch_decode(lowerCamelCase__  ,    skip_special_tokens=lowerCamelCase__		)

										_UpperCAmelCase				:						Any       						=			[
										    "Hello this is a long string of text.\n\nI\'m trying to get the text of the",
										    "Hey, I\'m a little late to the party. I\'m going to",
										]

										self.assertListEqual(lowerCamelCase__  ,    lowerCamelCase__		)


						@is_pt_flax_cross_test
						def 				_UpperCAmelCase (				self		):







										'''simple docstring'''


										_UpperCAmelCase						,							_UpperCAmelCase				:						Union[str, Any]       						=			self.model_tester.prepare_config_and_inputs_for_common()

										for model_class in self.all_model_classes:
														with self.subTest(model_class.__name__		):
																		# prepare inputs
																		_UpperCAmelCase				:						Union[str, Any]       						=			self._prepare_for_class(lowerCamelCase__  ,    lowerCamelCase__		)
																		_UpperCAmelCase				:						int       						=			{k: torch.tensor(v.tolist()		) for k, v in prepared_inputs_dict.items()}

																		# load corresponding PyTorch class
																		_UpperCAmelCase				:						Tuple       						=			model_class.__name__[4:]  # Skip the "Flax" at the beginning
																		_UpperCAmelCase				:						List[str]       						=			getattr(lowerCamelCase__  ,    lowerCamelCase__		)

																		_UpperCAmelCase						,							_UpperCAmelCase				:						Any       						=			pt_inputs["input_ids"].shape
																		_UpperCAmelCase				:						Optional[Any]       						=			np.random.randint(0  ,    seq_length - 1  ,    size=(batch_size,)		)
																		for batch_idx, start_index in enumerate(lowerCamelCase__		):
																						_UpperCAmelCase				:						int       						=			0
																						_UpperCAmelCase				:						Optional[Any]       						=			1
																						_UpperCAmelCase				:						List[str]       						=			0
																						_UpperCAmelCase				:						Any       						=			1
																		_UpperCAmelCase				:						int       						=			pt_model_class(lowerCamelCase__		).eval()
																		_UpperCAmelCase				:						Any       						=			model_class(lowerCamelCase__  ,    dtype=jnp.floataa		)

																		_UpperCAmelCase				:						Optional[Any]       						=			convert_pytorch_state_dict_to_flax(pt_model.state_dict()  ,    lowerCamelCase__		)
																		_UpperCAmelCase				:						Any       						=			fx_state

																		with torch.no_grad():
																						_UpperCAmelCase				:						Dict       						=			pt_model(**lowerCamelCase__		).to_tuple()

																		_UpperCAmelCase				:						str       						=			fx_model(**lowerCamelCase__		).to_tuple()
																		self.assertEqual(len(lowerCamelCase__		)  ,    len(lowerCamelCase__		)  ,    "Output lengths differ between Flax and PyTorch"		)
																		for fx_output, pt_output in zip(lowerCamelCase__  ,    lowerCamelCase__		):
																						self.assert_almost_equals(fx_output[:, -1]  ,    pt_output[:, -1].numpy()  ,    4e-2		)

																		with tempfile.TemporaryDirectory() as tmpdirname:
																						pt_model.save_pretrained(lowerCamelCase__		)
																						_UpperCAmelCase				:						List[str]       						=			model_class.from_pretrained(lowerCamelCase__  ,    from_pt=lowerCamelCase__		)

																		_UpperCAmelCase				:						Any       						=			fx_model_loaded(**lowerCamelCase__		).to_tuple()
																		self.assertEqual(
																		    len(lowerCamelCase__		)  ,    len(lowerCamelCase__		)  ,    "Output lengths differ between Flax and PyTorch"		)
																		for fx_output_loaded, pt_output in zip(lowerCamelCase__  ,    lowerCamelCase__		):
																						self.assert_almost_equals(fx_output_loaded[:, -1]  ,    pt_output[:, -1].numpy()  ,    4e-2		)


						@is_pt_flax_cross_test
						def 				_UpperCAmelCase (				self		):







										'''simple docstring'''


										_UpperCAmelCase						,							_UpperCAmelCase				:						List[Any]       						=			self.model_tester.prepare_config_and_inputs_for_common()
										for model_class in self.all_model_classes:
														with self.subTest(model_class.__name__		):
																		# prepare inputs
																		_UpperCAmelCase				:						Optional[int]       						=			self._prepare_for_class(lowerCamelCase__  ,    lowerCamelCase__		)
																		_UpperCAmelCase				:						Dict       						=			{k: torch.tensor(v.tolist()		) for k, v in prepared_inputs_dict.items()}

																		# load corresponding PyTorch class
																		_UpperCAmelCase				:						Optional[int]       						=			model_class.__name__[4:]  # Skip the "Flax" at the beginning
																		_UpperCAmelCase				:						Tuple       						=			getattr(lowerCamelCase__  ,    lowerCamelCase__		)

																		_UpperCAmelCase				:						Union[str, Any]       						=			pt_model_class(lowerCamelCase__		).eval()
																		_UpperCAmelCase				:						Optional[Any]       						=			model_class(lowerCamelCase__  ,    dtype=jnp.floataa		)

																		_UpperCAmelCase				:						List[Any]       						=			load_flax_weights_in_pytorch_model(lowerCamelCase__  ,    fx_model.params		)
																		_UpperCAmelCase						,							_UpperCAmelCase				:						List[Any]       						=			pt_inputs["input_ids"].shape
																		_UpperCAmelCase				:						Any       						=			np.random.randint(0  ,    seq_length - 1  ,    size=(batch_size,)		)
																		for batch_idx, start_index in enumerate(lowerCamelCase__		):
																						_UpperCAmelCase				:						Optional[int]       						=			0
																						_UpperCAmelCase				:						List[Any]       						=			1
																						_UpperCAmelCase				:						List[str]       						=			0
																						_UpperCAmelCase				:						int       						=			1

																		# make sure weights are tied in PyTorch
																		pt_model.tie_weights()

																		with torch.no_grad():
																						_UpperCAmelCase				:						Dict       						=			pt_model(**lowerCamelCase__		).to_tuple()

																		_UpperCAmelCase				:						List[str]       						=			fx_model(**lowerCamelCase__		).to_tuple()
																		self.assertEqual(len(lowerCamelCase__		)  ,    len(lowerCamelCase__		)  ,    "Output lengths differ between Flax and PyTorch"		)
																		for fx_output, pt_output in zip(lowerCamelCase__  ,    lowerCamelCase__		):
																						self.assert_almost_equals(fx_output[:, -1]  ,    pt_output[:, -1].numpy()  ,    4e-2		)

																		with tempfile.TemporaryDirectory() as tmpdirname:
																						fx_model.save_pretrained(lowerCamelCase__		)
																						_UpperCAmelCase				:						str       						=			pt_model_class.from_pretrained(lowerCamelCase__  ,    from_flax=lowerCamelCase__		)

																		with torch.no_grad():
																						_UpperCAmelCase				:						Tuple       						=			pt_model_loaded(**lowerCamelCase__		).to_tuple()

																		self.assertEqual(
																		    len(lowerCamelCase__		)  ,    len(lowerCamelCase__		)  ,    "Output lengths differ between Flax and PyTorch"		)
																		for fx_output, pt_output in zip(lowerCamelCase__  ,    lowerCamelCase__		):
																						self.assert_almost_equals(fx_output[:, -1]  ,    pt_output[:, -1].numpy()  ,    4e-2		)





						@slow
						def 				_UpperCAmelCase (				self		):







										'''simple docstring'''


										for model_class_name in self.all_model_classes:
														_UpperCAmelCase				:						Union[str, Any]       						=			model_class_name.from_pretrained("EleutherAI/gpt-neo-125M"		)
														_UpperCAmelCase				:						str       						=			model(np.ones((1, 1)		)		)
														self.assertIsNotNone(lowerCamelCase__		)







"""simple docstring"""
import argparse
import logging
import math
import time

import torch

from transformers import TransfoXLCorpus, TransfoXLLMHeadModel


logging.basicConfig(
    format='''%(asctime)s - %(levelname)s - %(name)s - %(message)s''', datefmt='''%m/%d/%Y %H:%M:%S''', level=logging.INFO
)
__SCREAMING_SNAKE_CASE     :  Dict										=					logging.getLogger(__name__)



def 				lowerCAmelCase_(      )		->				Optional[int]:
				_lowerCamelCase					     = argparse.ArgumentParser(description='''PyTorch Transformer Language Model'''							)
				parser.add_argument('''--model_name'''  ,      type=lowercase_  ,      default='''transfo-xl-wt103'''  ,      help='''pretrained model name'''							)
				parser.add_argument(
				    '''--split'''  ,      type=lowercase_  ,      default='''test'''  ,      choices=['''all''', '''valid''', '''test''']  ,      help='''which split to evaluate'''							)
				parser.add_argument('''--batch_size'''  ,      type=lowercase_  ,      default=10  ,      help='''batch size'''							)
				parser.add_argument('''--tgt_len'''  ,      type=lowercase_  ,      default=1_28  ,      help='''number of tokens to predict'''							)
				parser.add_argument('''--ext_len'''  ,      type=lowercase_  ,      default=0  ,      help='''length of the extended context'''							)
				parser.add_argument('''--mem_len'''  ,      type=lowercase_  ,      default=16_00  ,      help='''length of the retained previous heads'''							)
				parser.add_argument('''--clamp_len'''  ,      type=lowercase_  ,      default=10_00  ,      help='''max positional embedding index'''							)
				parser.add_argument('''--no_cuda'''  ,      action='''store_true'''  ,      help='''Do not use CUDA even though CUA is available'''							)
				parser.add_argument('''--work_dir'''  ,      type=lowercase_  ,      required=lowercase_  ,      help='''path to the work_dir'''							)
				parser.add_argument('''--no_log'''  ,      action='''store_true'''  ,      help='''do not log the eval result'''							)
				parser.add_argument('''--same_length'''  ,      action='''store_true'''  ,      help='''set same length attention with masking'''							)
				parser.add_argument('''--server_ip'''  ,      type=lowercase_  ,      default=''''''  ,      help='''Can be used for distant debugging.'''							)
				parser.add_argument('''--server_port'''  ,      type=lowercase_  ,      default=''''''  ,      help='''Can be used for distant debugging.'''							)
				_lowerCamelCase					     = parser.parse_args()
				assert args.ext_len >= 0, "extended context length must be non-negative"

				if args.server_ip and args.server_port:
								# Distant debugging - see https://code.visualstudio.com/docs/python/debugging#_attach-to-a-local-script
								import ptvsd

								print('''Waiting for debugger attach'''							)
								ptvsd.enable_attach(address=(args.server_ip, args.server_port)  ,      redirect_output=lowercase_							)
								ptvsd.wait_for_attach()

				_lowerCamelCase					     = torch.device('''cuda''' if torch.cuda.is_available() and not args.no_cuda else '''cpu'''							)
				logger.info('''device: {}'''.format(lowercase_							)							)

				# Load a pre-processed dataset
				# You can also build the corpus yourself using TransfoXLCorpus methods
				# The pre-processing involve computing word frequencies to prepare the Adaptive input and SoftMax
				# and tokenizing the dataset
				# The pre-processed corpus is a convertion (using the conversion script )
				_lowerCamelCase					     = TransfoXLCorpus.from_pretrained(args.model_name							)

				_lowerCamelCase					     = corpus.get_iterator('''valid'''  ,      args.batch_size  ,      args.tgt_len  ,      device=lowercase_  ,      ext_len=args.ext_len							)
				_lowerCamelCase					     = corpus.get_iterator('''test'''  ,      args.batch_size  ,      args.tgt_len  ,      device=lowercase_  ,      ext_len=args.ext_len							)

				# Load a pre-trained model
				_lowerCamelCase					     = TransfoXLLMHeadModel.from_pretrained(args.model_name							)
				model.to(lowercase_							)

				logger.info(
				    '''Evaluating with bsz {} tgt_len {} ext_len {} mem_len {} clamp_len {}'''.format(
				        args.batch_size  ,      args.tgt_len  ,      args.ext_len  ,      args.mem_len  ,      args.clamp_len							)							)

				model.reset_memory_length(args.mem_len							)
				if args.clamp_len > 0:
								_lowerCamelCase					     = args.clamp_len
				if args.same_length:
								_lowerCamelCase					     = True

				###############################################################################
				# Evaluation code
				###############################################################################
				def evaluate(lowercase_   :							Any							):
								# Turn on evaluation mode which disables dropout.
								model.eval()
								_lowerCamelCase     ,   _lowerCamelCase					     = 0, 0.0
								_lowerCamelCase					     = time.time()
								with torch.no_grad():
												_lowerCamelCase					     = None
												for idx, (data, target, seq_len) in enumerate(lowercase_							):
																_lowerCamelCase					     = model(lowercase_  ,      lm_labels=lowercase_  ,      mems=lowercase_							)
																_lowerCamelCase     ,   _lowerCamelCase     ,   _lowerCamelCase					     = ret
																_lowerCamelCase					     = loss.mean()
																total_loss += seq_len * loss.item()
																total_len += seq_len
												_lowerCamelCase					     = time.time() - start_time
								logger.info('''Time : {:.2f}s, {:.2f}ms/segment'''.format(lowercase_  ,      10_00 * total_time / (idx + 1)							)							)
								return total_loss / total_len

				# Run on test data.
				if args.split == "all":
								_lowerCamelCase					     = evaluate(lowercase_							)
								_lowerCamelCase					     = evaluate(lowercase_							)
				elif args.split == "valid":
								_lowerCamelCase					     = evaluate(lowercase_							)
								_lowerCamelCase					     = None
				elif args.split == "test":
								_lowerCamelCase					     = evaluate(lowercase_							)
								_lowerCamelCase					     = None

				def format_log(lowercase_   :							Optional[Any]  ,      lowercase_   :							Optional[Any]							):
								_lowerCamelCase					     = '''| {0} loss {1:5.2f} | {0} ppl {2:9.3f} '''.format(lowercase_  ,      lowercase_  ,      math.exp(lowercase_							)							)
								return log_str

				_lowerCamelCase					     = ''''''
				if valid_loss is not None:
								log_str += format_log(lowercase_  ,      '''valid'''							)
				if test_loss is not None:
								log_str += format_log(lowercase_  ,      '''test'''							)

				logger.info('''=''' * 1_00							)
				logger.info(lowercase_							)
				logger.info('''=''' * 1_00							)


if __name__ == "__main__":
			main()
