




"""simple docstring"""







import argparse
import json
import os
import pickle
import shutil

import numpy as np
import torch
from distiller import Distiller
from lm_seqs_dataset import LmSeqsDataset

from transformers import (
    BertConfig,
    BertForMaskedLM,
    BertTokenizer,
    DistilBertConfig,
    DistilBertForMaskedLM,
    DistilBertTokenizer,
    GPTaConfig,
    GPTaLMHeadModel,
    GPTaTokenizer,
    RobertaConfig,
    RobertaForMaskedLM,
    RobertaTokenizer,
)
from utils import git_log, init_gpu_params, logger, set_seed


a      =							{
    'distilbert': (DistilBertConfig, DistilBertForMaskedLM, DistilBertTokenizer),
    'roberta': (RobertaConfig, RobertaForMaskedLM, RobertaTokenizer),
    'bert': (BertConfig, BertForMaskedLM, BertTokenizer),
    'gpt2': (GPTaConfig, GPTaLMHeadModel, GPTaTokenizer),
}





def 							lowercase					(snake_case__   : List[str] )     ->  str:


				'''simple docstring'''






				assert (args.mlm and args.alpha_mlm > 0.0) or (not args.mlm and args.alpha_mlm == 0.0)
				assert (args.alpha_mlm > 0.0 and args.alpha_clm == 0.0) or (args.alpha_mlm == 0.0 and args.alpha_clm > 0.0)
				if args.mlm:
								assert os.path.isfile(args.token_counts )
								assert (args.student_type in ["roberta", "distilbert"]) and (args.teacher_type in ["roberta", "bert"])
				else:
								assert (args.student_type in ["gpt2"]) and (args.teacher_type in ["gpt2"])

				assert args.teacher_type == args.student_type or (
				    args.student_type == "distilbert" and args.teacher_type == "bert"
				)
				assert os.path.isfile(args.student_config )
				if args.student_pretrained_weights is not None:
								assert os.path.isfile(args.student_pretrained_weights )

				if args.freeze_token_type_embds:
								assert args.student_type in ["roberta"]

				assert args.alpha_ce >= 0.0
				assert args.alpha_mlm >= 0.0
				assert args.alpha_clm >= 0.0
				assert args.alpha_mse >= 0.0
				assert args.alpha_cos >= 0.0
				assert args.alpha_ce + args.alpha_mlm + args.alpha_clm + args.alpha_mse + args.alpha_cos > 0.0





def 							lowercase					(snake_case__   : List[str]      ,       snake_case__   : int )     ->  Any:


				'''simple docstring'''






				if args.student_type == "roberta":
								lowerCAmelCase  =				False
				elif args.student_type == "gpt2":
								lowerCAmelCase  =				False





def 							lowercase					(snake_case__   : str      ,       snake_case__   : str )     ->  Optional[Any]:


				'''simple docstring'''






				if args.student_type == "roberta":
								lowerCAmelCase  =				False





def 							lowercase					()     ->  Dict:


				'''simple docstring'''






				lowerCAmelCase  =				argparse.ArgumentParser(description="""Training""" )
				parser.add_argument("""--force"""      ,       action="""store_true"""      ,       help="""Overwrite dump_path if it already exists.""" )

				parser.add_argument(
				    """--dump_path"""      ,       type=snake_case__      ,       required=snake_case__      ,       help="""The output directory (log, checkpoints, parameters, etc.)""" )
				parser.add_argument(
				    """--data_file"""      ,       type=snake_case__      ,       required=snake_case__      ,       help="""The binarized file (tokenized + tokens_to_ids) and grouped by sequence."""      ,       )

				parser.add_argument(
				    """--student_type"""      ,       type=snake_case__      ,       choices=["""distilbert""", """roberta""", """gpt2"""]      ,       required=snake_case__      ,       help="""The student type (DistilBERT, RoBERTa)."""      ,       )
				parser.add_argument("""--student_config"""      ,       type=snake_case__      ,       required=snake_case__      ,       help="""Path to the student configuration.""" )
				parser.add_argument(
				    """--student_pretrained_weights"""      ,       default=snake_case__      ,       type=snake_case__      ,       help="""Load student initialization checkpoint.""" )

				parser.add_argument(
				    """--teacher_type"""      ,       choices=["""bert""", """roberta""", """gpt2"""]      ,       required=snake_case__      ,       help="""Teacher type (BERT, RoBERTa).""" )
				parser.add_argument("""--teacher_name"""      ,       type=snake_case__      ,       required=snake_case__      ,       help="""The teacher model.""" )

				parser.add_argument("""--temperature"""      ,       default=2.0      ,       type=snake_case__      ,       help="""Temperature for the softmax temperature.""" )
				parser.add_argument(
				    """--alpha_ce"""      ,       default=0.5      ,       type=snake_case__      ,       help="""Linear weight for the distillation loss. Must be >=0.""" )
				parser.add_argument(
				    """--alpha_mlm"""      ,       default=0.0      ,       type=snake_case__      ,       help="""Linear weight for the MLM loss. Must be >=0. Should be used in conjunction with `mlm` flag."""      ,       )
				parser.add_argument("""--alpha_clm"""      ,       default=0.5      ,       type=snake_case__      ,       help="""Linear weight for the CLM loss. Must be >=0.""" )
				parser.add_argument("""--alpha_mse"""      ,       default=0.0      ,       type=snake_case__      ,       help="""Linear weight of the MSE loss. Must be >=0.""" )
				parser.add_argument(
				    """--alpha_cos"""      ,       default=0.0      ,       type=snake_case__      ,       help="""Linear weight of the cosine embedding loss. Must be >=0.""" )

				parser.add_argument(
				    """--mlm"""      ,       action="""store_true"""      ,       help="""The LM step: MLM or CLM. If `mlm` is True, the MLM is used over CLM.""" )
				parser.add_argument(
				    """--mlm_mask_prop"""      ,       default=0.15      ,       type=snake_case__      ,       help="""Proportion of tokens for which we need to make a prediction."""      ,       )
				parser.add_argument("""--word_mask"""      ,       default=0.8      ,       type=snake_case__      ,       help="""Proportion of tokens to mask out.""" )
				parser.add_argument("""--word_keep"""      ,       default=0.1      ,       type=snake_case__      ,       help="""Proportion of tokens to keep.""" )
				parser.add_argument("""--word_rand"""      ,       default=0.1      ,       type=snake_case__      ,       help="""Proportion of tokens to randomly replace.""" )
				parser.add_argument(
				    """--mlm_smoothing"""      ,       default=0.7      ,       type=snake_case__      ,       help="""Smoothing parameter to emphasize more rare tokens (see XLM, similar to word2vec)."""      ,       )
				parser.add_argument("""--token_counts"""      ,       type=snake_case__      ,       help="""The token counts in the data_file for MLM.""" )

				parser.add_argument(
				    """--restrict_ce_to_mask"""      ,       action="""store_true"""      ,       help="""If true, compute the distillation loss only the [MLM] prediction distribution."""      ,       )
				parser.add_argument(
				    """--freeze_pos_embs"""      ,       action="""store_true"""      ,       help="""Freeze positional embeddings during distillation. For student_type in ['roberta', 'gpt2'] only."""      ,       )
				parser.add_argument(
				    """--freeze_token_type_embds"""      ,       action="""store_true"""      ,       help="""Freeze token type embeddings during distillation if existent. For student_type in ['roberta'] only."""      ,       )

				parser.add_argument("""--n_epoch"""      ,       type=snake_case__      ,       default=3      ,       help="""Number of pass on the whole dataset.""" )
				parser.add_argument("""--batch_size"""      ,       type=snake_case__      ,       default=5      ,       help="""Batch size (for each process).""" )
				parser.add_argument(
				    """--group_by_size"""      ,       action="""store_false"""      ,       help="""If true, group sequences that have similar length into the same batch. Default is true."""      ,       )

				parser.add_argument(
				    """--gradient_accumulation_steps"""      ,       type=snake_case__      ,       default=50      ,       help="""Gradient accumulation for larger training batches."""      ,       )
				parser.add_argument("""--warmup_prop"""      ,       default=0.05      ,       type=snake_case__      ,       help="""Linear warmup proportion.""" )
				parser.add_argument("""--weight_decay"""      ,       default=0.0      ,       type=snake_case__      ,       help="""Weight decay if we apply some.""" )
				parser.add_argument("""--learning_rate"""      ,       default=5e-4      ,       type=snake_case__      ,       help="""The initial learning rate for Adam.""" )
				parser.add_argument("""--adam_epsilon"""      ,       default=1e-6      ,       type=snake_case__      ,       help="""Epsilon for Adam optimizer.""" )
				parser.add_argument("""--max_grad_norm"""      ,       default=5.0      ,       type=snake_case__      ,       help="""Max gradient norm.""" )
				parser.add_argument("""--initializer_range"""      ,       default=0.02      ,       type=snake_case__      ,       help="""Random initialization range.""" )

				parser.add_argument(
				    """--fp16"""      ,       action="""store_true"""      ,       help="""Whether to use 16-bit (mixed) precision (through NVIDIA apex) instead of 32-bit"""      ,       )
				parser.add_argument(
				    """--fp16_opt_level"""      ,       type=snake_case__      ,       default="""O1"""      ,       help=(
				        """For fp16: Apex AMP optimization level selected in ['O0', 'O1', 'O2', and 'O3']."""
				        """See details at https://nvidia.github.io/apex/amp.html"""
				    )      ,       )
				parser.add_argument("""--n_gpu"""      ,       type=snake_case__      ,       default=1      ,       help="""Number of GPUs in the node.""" )
				parser.add_argument("""--local_rank"""      ,       type=snake_case__      ,       default=-1      ,       help="""Distributed training - Local rank""" )
				parser.add_argument("""--seed"""      ,       type=snake_case__      ,       default=56      ,       help="""Random seed""" )

				parser.add_argument("""--log_interval"""      ,       type=snake_case__      ,       default=500      ,       help="""Tensorboard logging interval.""" )
				parser.add_argument("""--checkpoint_interval"""      ,       type=snake_case__      ,       default=4_000      ,       help="""Checkpoint interval.""" )
				lowerCAmelCase  =				parser.parse_args()
				sanity_checks(snake_case__ )

				# ARGS #
				init_gpu_params(snake_case__ )
				set_seed(snake_case__ )
				if args.is_master:
								if os.path.exists(args.dump_path ):
												if not args.force:
																raise ValueError(
																    f'''Serialization dir {args.dump_path} already exists, but you have not precised wheter to overwrite'''
																    """ itUse `--force` if you want to overwrite it""" )
												else:
																shutil.rmtree(args.dump_path )

								if not os.path.exists(args.dump_path ):
												os.makedirs(args.dump_path )
								logger.info(f'''Experiment will be dumped and logged in {args.dump_path}''' )

								# SAVE PARAMS #
								logger.info(f'''Param: {args}''' )
								with open(os.path.join(args.dump_path      ,       """parameters.json""" )      ,       """w""" ) as f:
												json.dump(vars(snake_case__ )      ,       snake_case__      ,       indent=4 )
								git_log(args.dump_path )

				lowerCAmelCase ,						lowerCAmelCase ,						lowerCAmelCase  =				MODEL_CLASSES[args.student_type]
				lowerCAmelCase ,						lowerCAmelCase ,						lowerCAmelCase  =				MODEL_CLASSES[args.teacher_type]

				# TOKENIZER #
				lowerCAmelCase  =				teacher_tokenizer_class.from_pretrained(args.teacher_name )
				lowerCAmelCase  =				{}
				for tok_name, tok_symbol in tokenizer.special_tokens_map.items():
								lowerCAmelCase  =				tokenizer.all_special_tokens.index(snake_case__ )
								lowerCAmelCase  =				tokenizer.all_special_ids[idx]
				logger.info(f'''Special tokens {special_tok_ids}''' )
				lowerCAmelCase  =				special_tok_ids
				lowerCAmelCase  =				tokenizer.max_model_input_sizes[args.teacher_name]

				# DATA LOADER #
				logger.info(f'''Loading data from {args.data_file}''' )
				with open(args.data_file      ,       """rb""" ) as fp:
								lowerCAmelCase  =				pickle.load(snake_case__ )

				if args.mlm:
								logger.info(f'''Loading token counts from {args.token_counts} (already pre-computed)''' )
								with open(args.token_counts      ,       """rb""" ) as fp:
												lowerCAmelCase  =				pickle.load(snake_case__ )

								lowerCAmelCase  =				np.maximum(snake_case__      ,       1 ) ** -args.mlm_smoothing
								for idx in special_tok_ids.values():
												lowerCAmelCase  =				0.0  # do not predict special tokens
								lowerCAmelCase  =				torch.from_numpy(snake_case__ )
				else:
								lowerCAmelCase  =				None

				lowerCAmelCase  =				LmSeqsDataset(params=snake_case__      ,       data=snake_case__ )
				logger.info("""Data loader created.""" )

				# STUDENT #
				logger.info(f'''Loading student config from {args.student_config}''' )
				lowerCAmelCase  =				student_config_class.from_pretrained(args.student_config )
				lowerCAmelCase  =				True

				if args.student_pretrained_weights is not None:
								logger.info(f'''Loading pretrained weights from {args.student_pretrained_weights}''' )
								lowerCAmelCase  =				student_model_class.from_pretrained(args.student_pretrained_weights      ,       config=snake_case__ )
				else:
								lowerCAmelCase  =				student_model_class(snake_case__ )

				if args.n_gpu > 0:
								student.to(f'''cuda:{args.local_rank}''' )
				logger.info("""Student loaded.""" )

				# TEACHER #
				lowerCAmelCase  =				teacher_model_class.from_pretrained(args.teacher_name      ,       output_hidden_states=snake_case__ )
				if args.n_gpu > 0:
								teacher.to(f'''cuda:{args.local_rank}''' )
				logger.info(f'''Teacher loaded from {args.teacher_name}.''' )

				# FREEZING #
				if args.freeze_pos_embs:
								freeze_pos_embeddings(snake_case__      ,       snake_case__ )
				if args.freeze_token_type_embds:
								freeze_token_type_embeddings(snake_case__      ,       snake_case__ )

				# SANITY CHECKS #
				assert student.config.vocab_size == teacher.config.vocab_size
				assert student.config.hidden_size == teacher.config.hidden_size
				assert student.config.max_position_embeddings == teacher.config.max_position_embeddings
				if args.mlm:
								assert token_probs.size(0 ) == stu_architecture_config.vocab_size

				# DISTILLER #
				torch.cuda.empty_cache()
				lowerCAmelCase  =				Distiller(
				    params=snake_case__      ,       dataset=snake_case__      ,       token_probs=snake_case__      ,       student=snake_case__      ,       teacher=snake_case__ )
				distiller.train()
				logger.info("""Let's go get some drinks.""" )


if __name__ == "__main__":
		main()











"""simple docstring"""







import random
import timeit
from functools import wraps
from typing import Callable, Optional

from ..configuration_utils import PretrainedConfig
from ..models.auto.modeling_tf_auto import TF_MODEL_MAPPING, TF_MODEL_WITH_LM_HEAD_MAPPING
from ..utils import is_pyanvml_available, is_tf_available, logging
from .benchmark_utils import (
    Benchmark,
    Memory,
    MemorySummary,
    measure_peak_memory_cpu,
    start_memory_tracing,
    stop_memory_tracing,
)


if is_tf_available():
		import tensorflow as tf
		from tensorflow.python.framework.errors_impl import ResourceExhaustedError

		from .benchmark_args_tf import TensorFlowBenchmarkArguments

if is_pyanvml_available():
		import pyanvml.pyanvml as nvml

a      =							logging.get_logger(__name__)





def 							lowercase					(snake_case__   : bool      ,       snake_case__   : bool )     ->  Tuple:


				'''simple docstring'''






				def run_func(snake_case__   : Any ):
								@wraps(snake_case__ )
								def run_in_eager_mode(*snake_case__   : Optional[Any]      ,       **snake_case__   : int ):
												return func(*snake_case__      ,       **snake_case__ )

								@wraps(snake_case__ )
								@tf.function(experimental_compile=snake_case__ )
								def run_in_graph_mode(*snake_case__   : int      ,       **snake_case__   : Tuple ):
												return func(*snake_case__      ,       **snake_case__ )

								if do_eager_mode is True:
												if use_xla is not False:
																raise ValueError(
																    """Cannot run model in XLA, if `args.eager_mode` is set to `True`. Please set `args.eager_mode=False`.""" )
												return run_in_eager_mode
								else:
												return run_in_graph_mode

				return run_func





def 							lowercase					(snake_case__   : int      ,       snake_case__   : int      ,       snake_case__   : int )     ->  ["tf.Tensor"]:


				'''simple docstring'''






				lowerCAmelCase  =				random.Random()
				lowerCAmelCase  =				[rng.randint(0      ,       vocab_size - 1 ) for i in range(batch_size * sequence_length )]
				return tf.constant(snake_case__      ,       shape=(batch_size, sequence_length)      ,       dtype=tf.intaa )






class       SCREAMING_SNAKE_CASE__					(					_a					):
						_a    						=						42
						_a    						=						42
						_a    						=						"TensorFlow"


						@property
						def   __lowercase (							self				:			Optional[int]					):
										return tf.__version__


						def   __lowercase (							self				:			Optional[Any]						,						lowerCAmelCase				:			str						,						lowerCAmelCase				:			int						,						lowerCAmelCase				:			int					):
										# initialize GPU on separate process
										lowerCAmelCase  =				self.args.strategy
										if strategy is None:
														raise ValueError("""A device strategy has to be initialized before using TensorFlow."""					)
										lowerCAmelCase  =				self._prepare_inference_func(lowerCAmelCase						,						lowerCAmelCase						,						lowerCAmelCase					)
										return self._measure_speed(_inference					)


						def   __lowercase (							self				:			str						,						lowerCAmelCase				:			str						,						lowerCAmelCase				:			int						,						lowerCAmelCase				:			int					):
										lowerCAmelCase  =				self.args.strategy
										if strategy is None:
														raise ValueError("""A device strategy has to be initialized before using TensorFlow."""					)
										lowerCAmelCase  =				self._prepare_train_func(lowerCAmelCase						,						lowerCAmelCase						,						lowerCAmelCase					)
										return self._measure_speed(_train					)


						def   __lowercase (							self				:			str						,						lowerCAmelCase				:			str						,						lowerCAmelCase				:			int						,						lowerCAmelCase				:			int					):
										# initialize GPU on separate process
										if self.args.is_gpu:
														tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx]						,						lowerCAmelCase					)
										lowerCAmelCase  =				self.args.strategy
										if strategy is None:
														raise ValueError("""A device strategy has to be initialized before using TensorFlow."""					)
										lowerCAmelCase  =				self._prepare_inference_func(lowerCAmelCase						,						lowerCAmelCase						,						lowerCAmelCase					)
										return self._measure_memory(_inference					)


						def   __lowercase (							self				:			int						,						lowerCAmelCase				:			str						,						lowerCAmelCase				:			int						,						lowerCAmelCase				:			int					):
										if self.args.is_gpu:
														tf.config.experimental.set_memory_growth(self.args.gpu_list[self.args.device_idx]						,						lowerCAmelCase					)
										lowerCAmelCase  =				self.args.strategy
										if strategy is None:
														raise ValueError("""A device strategy has to be initialized before using TensorFlow."""					)

										lowerCAmelCase  =				self._prepare_train_func(lowerCAmelCase						,						lowerCAmelCase						,						lowerCAmelCase					)
										return self._measure_memory(_train					)


						def   __lowercase (							self				:			List[str]						,						lowerCAmelCase				:			str						,						lowerCAmelCase				:			int						,						lowerCAmelCase				:			int					):
										lowerCAmelCase  =				self.config_dict[model_name]

										if self.args.fpaa:
														raise NotImplementedError("""Mixed precision is currently not supported."""					)

										lowerCAmelCase  =				(
										    hasattr(lowerCAmelCase						,						"""architectures"""					)
										    and isinstance(config.architectures						,						lowerCAmelCase					)
										    and len(config.architectures					) > 0
										)
										if not self.args.only_pretrain_model and has_model_class_in_config:
														try:
																		lowerCAmelCase  =				"""TF""" + config.architectures[0]  # prepend 'TF' for tensorflow model
																		lowerCAmelCase  =				__import__("""transformers"""						,						fromlist=[model_class]					)
																		lowerCAmelCase  =				getattr(lowerCAmelCase						,						lowerCAmelCase					)
																		lowerCAmelCase  =				model_cls(lowerCAmelCase					)
														except ImportError:
																		raise ImportError(
																		    f'''{model_class} does not exist. If you just want to test the pretrained model, you might want to'''
																		    """ set `--only_pretrain_model` or `args.only_pretrain_model=True`."""					)
										else:
														lowerCAmelCase  =				TF_MODEL_MAPPING[config.__class__](lowerCAmelCase					)

										# encoder-decoder has vocab size saved differently
										lowerCAmelCase  =				config.vocab_size if hasattr(lowerCAmelCase						,						"""vocab_size"""					) else config.encoder.vocab_size
										lowerCAmelCase  =				random_input_ids(lowerCAmelCase						,						lowerCAmelCase						,						lowerCAmelCase					)

										@run_with_tf_optimizations(self.args.eager_mode						,						self.args.use_xla					)
										def encoder_decoder_forward():
														return model(lowerCAmelCase						,						decoder_input_ids=lowerCAmelCase						,						training=lowerCAmelCase					)

										@run_with_tf_optimizations(self.args.eager_mode						,						self.args.use_xla					)
										def encoder_forward():
														return model(lowerCAmelCase						,						training=lowerCAmelCase					)

										lowerCAmelCase  =				encoder_decoder_forward if config.is_encoder_decoder else encoder_forward

										return _inference


						def   __lowercase (							self				:			Optional[int]						,						lowerCAmelCase				:			str						,						lowerCAmelCase				:			int						,						lowerCAmelCase				:			int					):
										lowerCAmelCase  =				self.config_dict[model_name]

										if self.args.eager_mode is not False:
														raise ValueError("""Training cannot be done in eager mode. Please make sure that `args.eager_mode = False`."""					)

										if self.args.fpaa:
														raise NotImplementedError("""Mixed precision is currently not supported."""					)

										lowerCAmelCase  =				(
										    hasattr(lowerCAmelCase						,						"""architectures"""					)
										    and isinstance(config.architectures						,						lowerCAmelCase					)
										    and len(config.architectures					) > 0
										)
										if not self.args.only_pretrain_model and has_model_class_in_config:
														try:
																		lowerCAmelCase  =				"""TF""" + config.architectures[0]  # prepend 'TF' for tensorflow model
																		lowerCAmelCase  =				__import__("""transformers"""						,						fromlist=[model_class]					)
																		lowerCAmelCase  =				getattr(lowerCAmelCase						,						lowerCAmelCase					)
																		lowerCAmelCase  =				model_cls(lowerCAmelCase					)
														except ImportError:
																		raise ImportError(
																		    f'''{model_class} does not exist. If you just want to test the pretrained model, you might want to'''
																		    """ set `--only_pretrain_model` or `args.only_pretrain_model=True`."""					)
										else:
														lowerCAmelCase  =				TF_MODEL_WITH_LM_HEAD_MAPPING[config.__class__](lowerCAmelCase					)

										# encoder-decoder has vocab size saved differently
										lowerCAmelCase  =				config.vocab_size if hasattr(lowerCAmelCase						,						"""vocab_size"""					) else config.encoder.vocab_size
										lowerCAmelCase  =				random_input_ids(lowerCAmelCase						,						lowerCAmelCase						,						lowerCAmelCase					)

										@run_with_tf_optimizations(self.args.eager_mode						,						self.args.use_xla					)
										def encoder_decoder_train():
														lowerCAmelCase  =				model(lowerCAmelCase						,						decoder_input_ids=lowerCAmelCase						,						labels=lowerCAmelCase						,						training=lowerCAmelCase					)[0]
														lowerCAmelCase  =				tf.gradients(lowerCAmelCase						,						model.trainable_variables					)
														return gradients

										@run_with_tf_optimizations(self.args.eager_mode						,						self.args.use_xla					)
										def encoder_train():
														lowerCAmelCase  =				model(lowerCAmelCase						,						labels=lowerCAmelCase						,						training=lowerCAmelCase					)[0]
														lowerCAmelCase  =				tf.gradients(lowerCAmelCase						,						model.trainable_variables					)
														return gradients

										lowerCAmelCase  =				encoder_decoder_train if config.is_encoder_decoder else encoder_train

										return _train


						def   __lowercase (							self				:			Optional[int]						,						lowerCAmelCase				:			List[str]					):
										with self.args.strategy.scope():
														try:
																		if self.args.is_tpu or self.args.use_xla:
																						# run additional 10 times to stabilize compilation for tpu
																						logger.info("""Do inference on TPU. Running model 5 times to stabilize compilation"""					)
																						timeit.repeat(lowerCAmelCase						,						repeat=1						,						number=5					)

																		# as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
																		lowerCAmelCase  =				timeit.repeat(
																		    lowerCAmelCase						,						repeat=self.args.repeat						,						number=10						,						)

																		return min(lowerCAmelCase					) / 10.0
														except ResourceExhaustedError as e:
																		self.print_fn(f'''Doesn\'t fit on GPU. {e}'''					)




						def   __lowercase (							self				:			Optional[Any]						,						lowerCAmelCase				:			Callable[[], None]					):
										logger.info(
										    """Note that TensorFlow allocates more memory than """
										    """it might need to speed up computation. """
										    """The memory reported here corresponds to the memory """
										    """reported by `nvidia-smi`, which can vary depending """
										    """on total available memory on the GPU that is used."""					)
										with self.args.strategy.scope():
														try:
																		if self.args.trace_memory_line_by_line:
																						if not self.args.eager_mode:
																										raise ValueError(
																										    """`args.eager_mode` is set to `False`. Make sure to run model in eager mode to measure memory"""
																										    """ consumption line by line."""					)
																						lowerCAmelCase  =				start_memory_tracing("""transformers"""					)

																		if self.args.is_tpu:
																						# tpu
																						raise NotImplementedError(
																						    """Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking"""
																						    """ with `args.memory=False`"""					)
																		elif self.args.is_gpu:
																						# gpu
																						if not is_pyanvml_available():
																										logger.warning(
																										    """py3nvml not installed, we won't log GPU memory usage. """
																										    """Install py3nvml (pip install py3nvml) to log information about GPU."""					)
																										lowerCAmelCase  =				"""N/A"""
																						else:
																										logger.info(
																										    """Measuring total GPU usage on GPU device. Make sure to not have additional processes"""
																										    """ running on the same GPU."""					)
																										# init nvml
																										nvml.nvmlInit()
																										func()
																										lowerCAmelCase  =				nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx					)
																										lowerCAmelCase  =				nvml.nvmlDeviceGetMemoryInfo(lowerCAmelCase					)
																										lowerCAmelCase  =				meminfo.used
																										lowerCAmelCase  =				Memory(lowerCAmelCase					)
																										# shutdown nvml
																										nvml.nvmlShutdown()
																		else:
																						# cpu
																						if self.args.trace_memory_line_by_line:
																										logger.info(
																										    """When enabling line by line tracing, the max peak memory for CPU is inaccurate in"""
																										    """ TensorFlow."""					)
																										lowerCAmelCase  =				None
																						else:
																										lowerCAmelCase  =				measure_peak_memory_cpu(lowerCAmelCase					)
																										lowerCAmelCase  =				Memory(lowerCAmelCase					) if isinstance(lowerCAmelCase						,						lowerCAmelCase					) else memory_bytes
																		if self.args.trace_memory_line_by_line:
																						lowerCAmelCase  =				stop_memory_tracing(lowerCAmelCase					)
																						if memory is None:
																										lowerCAmelCase  =				summary.total
																		else:
																						lowerCAmelCase  =				None

																		return memory, summary
														except ResourceExhaustedError as e:
																		self.print_fn(f'''Doesn\'t fit on GPU. {e}'''					)
																		return "N/A", None





