





'''simple docstring'''

import timeit
from typing import Callable, Optional

from ..configuration_utils import PretrainedConfig
from ..models.auto.modeling_auto import MODEL_MAPPING, MODEL_WITH_LM_HEAD_MAPPING
from ..utils import is_pyanvml_available, is_torch_available, logging
from .benchmark_utils import (
    Benchmark,
    Memory,
    MemorySummary,
    measure_peak_memory_cpu,
    start_memory_tracing,
    stop_memory_tracing,
)


if is_torch_available():
			import torch

			from .benchmark_args import PyTorchBenchmarkArguments


if is_pyanvml_available():
			import pyanvml.pyanvml as nvml


lowerCAmelCase__	:					Dict	     =       logging.get_logger(__name__)


class 			SCREAMING_SNAKE_CASE__	(					_a						):







					"""simple docstring"""






					SCREAMING_SNAKE_CASE				=     42
					SCREAMING_SNAKE_CASE				=     42
					SCREAMING_SNAKE_CASE				=     '''PyTorch'''
					@property
					def 						lowerCamelCase_					(					self    :	Optional[Any]						):







							"""simple docstring"""
							return torch.__version__
					def 						lowerCamelCase_					(					self    :	int						,     UpperCAmelCase_    :	str						,     UpperCAmelCase_    :	int						,     UpperCAmelCase_    :	int						):







							"""simple docstring"""
							__UpperCAmelCase     :				List[Any]					   =			self._prepare_inference_func(lowerCAmelCase_						,     lowerCAmelCase_						,     lowerCAmelCase_						)
							return self._measure_speed(_inference						)
					def 						lowerCamelCase_					(					self    :	List[Any]						,     UpperCAmelCase_    :	str						,     UpperCAmelCase_    :	int						,     UpperCAmelCase_    :	int						):







							"""simple docstring"""
							__UpperCAmelCase     :				Dict					   =			self._prepare_inference_func(lowerCAmelCase_						,     lowerCAmelCase_						,     lowerCAmelCase_						)
							return self._measure_memory(_inference						)
					def 						lowerCamelCase_					(					self    :	Union[str, Any]						,     UpperCAmelCase_    :	str						,     UpperCAmelCase_    :	int						,     UpperCAmelCase_    :	int						):







							"""simple docstring"""
							__UpperCAmelCase     :				Optional[int]					   =			self._prepare_train_func(lowerCAmelCase_						,     lowerCAmelCase_						,     lowerCAmelCase_						)
							return self._measure_speed(_train						)
					def 						lowerCamelCase_					(					self    :	int						,     UpperCAmelCase_    :	str						,     UpperCAmelCase_    :	int						,     UpperCAmelCase_    :	int						):







							"""simple docstring"""
							__UpperCAmelCase     :				Optional[Any]					   =			self._prepare_train_func(lowerCAmelCase_						,     lowerCAmelCase_						,     lowerCAmelCase_						)
							return self._measure_memory(_train						)
					def 						lowerCamelCase_					(					self    :	Union[str, Any]						,     UpperCAmelCase_    :	str						,     UpperCAmelCase_    :	int						,     UpperCAmelCase_    :	int						):







							"""simple docstring"""
							__UpperCAmelCase     :				Optional[Any]					   =			self.config_dict[model_name]

							if self.args.torchscript:
									__UpperCAmelCase     :				Optional[Any]					   =			True

							__UpperCAmelCase     :				Any					   =			(
							    hasattr(lowerCAmelCase_						,     "architectures"						)
							    and isinstance(config.architectures						,     lowerCAmelCase_						)
							    and len(config.architectures						) > 0
							)
							if not self.args.only_pretrain_model and has_model_class_in_config:
									try:
											__UpperCAmelCase     :				Tuple					   =			config.architectures[0]
											__UpperCAmelCase     :				Optional[Any]					   =			__import__("transformers"						,     fromlist=[model_class]						)
											__UpperCAmelCase     :				Dict					   =			getattr(lowerCAmelCase_						,     lowerCAmelCase_						)
											__UpperCAmelCase     :				List[Any]					   =			model_cls(lowerCAmelCase_						)
									except ImportError:
											raise ImportError(
											    f"{model_class} does not exist. If you just want to test the pretrained model, you might want to"
											    " set `--only_pretrain_model` or `args.only_pretrain_model=True`."						)
							else:
									__UpperCAmelCase     :				str					   =			MODEL_MAPPING[config.__class__](lowerCAmelCase_						)

							model.eval()
							model.to(self.args.device						)

							# encoder-decoder has vocab size saved differently
							__UpperCAmelCase     :				List[str]					   =			config.vocab_size if hasattr(lowerCAmelCase_						,     "vocab_size"						) else config.encoder.vocab_size
							__UpperCAmelCase     :				int					   =			torch.randint(lowerCAmelCase_						,     (batch_size, sequence_length)						,     dtype=torch.long						,     device=self.args.device						)

							if self.args.fpaa:
									logger.info("Running training in Mixed Precision..."						)
									if not self.args.is_gpu:
											raise ValueError("Mixed precision is possible only for GPU."						)
									# amp seems to have memory leaks so that memory usage
									# is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
									model.half()

							if self.args.torchscript:
									with torch.no_grad():
											__UpperCAmelCase     :				Tuple					   =			torch.jit.trace(lowerCAmelCase_						,     lowerCAmelCase_						)
							else:
									__UpperCAmelCase     :				Optional[Any]					   =			model

							def encoder_decoder_forward():
									with torch.no_grad():
											__UpperCAmelCase     :				Tuple					   =			inference_model(lowerCAmelCase_						,     decoder_input_ids=lowerCAmelCase_						)
									return outputs

							def encoder_forward():
									with torch.no_grad():
											__UpperCAmelCase     :				Any					   =			inference_model(lowerCAmelCase_						)
									return outputs

							__UpperCAmelCase     :				List[str]					   =			encoder_decoder_forward if config.is_encoder_decoder else encoder_forward
							return _forward
					def 						lowerCamelCase_					(					self    :	List[str]						,     UpperCAmelCase_    :	str						,     UpperCAmelCase_    :	int						,     UpperCAmelCase_    :	int						):







							"""simple docstring"""
							__UpperCAmelCase     :				Dict					   =			self.config_dict[model_name]

							__UpperCAmelCase     :				List[Any]					   =			(
							    hasattr(lowerCAmelCase_						,     "architectures"						)
							    and isinstance(config.architectures						,     lowerCAmelCase_						)
							    and len(config.architectures						) > 0
							)
							if not self.args.only_pretrain_model and has_model_class_in_config:
									try:
											__UpperCAmelCase     :				Union[str, Any]					   =			config.architectures[0]
											__UpperCAmelCase     :				Union[str, Any]					   =			__import__("transformers"						,     fromlist=[model_class]						)
											__UpperCAmelCase     :				Union[str, Any]					   =			getattr(lowerCAmelCase_						,     lowerCAmelCase_						)
											__UpperCAmelCase     :				Union[str, Any]					   =			model_cls(lowerCAmelCase_						)
									except ImportError:
											raise ImportError(
											    f"{model_class} does not exist. If you just want to test the pretrained model, you might want to"
											    " set `--only_pretrain_model` or `args.only_pretrain_model=True`."						)
							else:
									__UpperCAmelCase     :				str					   =			MODEL_WITH_LM_HEAD_MAPPING[config.__class__](lowerCAmelCase_						)

							if self.args.torchscript:
									raise NotImplementedError("Training for torchscript is currently not implemented"						)
							else:
									__UpperCAmelCase     :				Any					   =			model

							model.train()
							model.to(self.args.device						)

							# encoder-decoder has vocab size saved differently
							__UpperCAmelCase     :				Union[str, Any]					   =			config.vocab_size if hasattr(lowerCAmelCase_						,     "vocab_size"						) else config.encoder.vocab_size
							__UpperCAmelCase     :				Dict					   =			torch.randint(lowerCAmelCase_						,     (batch_size, sequence_length)						,     dtype=torch.long						,     device=self.args.device						)

							if self.args.fpaa:
									logger.info("Running training in Mixed Precision..."						)
									if not self.args.is_gpu:
											raise ValueError("Mixed precision is possible only for GPU."						)

									# amp seems to have memory leaks so that memory usage
									# is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
									model.half()

							def compute_loss_and_backprob_encoder():
									__UpperCAmelCase     :				Any					   =			train_model(lowerCAmelCase_						,     labels=lowerCAmelCase_						)[0]
									loss.backward()
									return loss

							def compute_loss_and_backprob_encoder_decoder():
									__UpperCAmelCase     :				int					   =			train_model(lowerCAmelCase_						,     decoder_input_ids=lowerCAmelCase_						,     labels=lowerCAmelCase_						)[0]
									loss.backward()
									return loss

							__UpperCAmelCase     :				List[str]					   =			(
							    compute_loss_and_backprob_encoder_decoder
							    if config.is_encoder_decoder
							    else compute_loss_and_backprob_encoder
							)
							return _train
					def 						lowerCamelCase_					(					self    :	int						,     UpperCAmelCase_    :	Dict						):







							"""simple docstring"""


							try:
									if self.args.is_tpu or self.args.torchscript:
											# run additional 10 times to stabilize compilation for tpu and torchscript
											logger.info("Do inference on TPU or torchscript. Running model 5 times to stabilize compilation"						)
											timeit.repeat(
											    lowerCAmelCase_						,     repeat=1						,     number=5						,     )

									# as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
									__UpperCAmelCase     :				List[Any]					   =			timeit.repeat(
									    lowerCAmelCase_						,     repeat=self.args.repeat						,     number=10						,     )

									if self.args.is_tpu and self.args.torch_xla_tpu_print_metrics:
											import torch_xla.debug.metrics as met

											self.print_fn(met.metrics_report()						)

									return min(lowerCAmelCase_						) / 10.0
							except RuntimeError as e:
									self.print_fn(f"Doesn\'t fit on GPU. {e}"						)
									return "N/A"






					def 						lowerCamelCase_					(					self    :	int						,     UpperCAmelCase_    :	Callable[[], None]						):







							"""simple docstring"""


							try:
									if self.args.trace_memory_line_by_line:
											__UpperCAmelCase     :				Optional[int]					   =			start_memory_tracing("transformers"						)

									if self.args.is_tpu:
											# tpu
											raise NotImplementedError(
											    "Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking with"
											    " `--no-memory` or `args.memory=False`"						)
									elif self.args.is_gpu:
											if not is_pyanvml_available():
													logger.warning(
													    "py3nvml not installed, we won\'t log GPU memory usage. "
													    "Install py3nvml (pip install py3nvml) to log information about GPU."						)
													__UpperCAmelCase     :				str					   =			"N/A"
											else:
													logger.info(
													    "Measuring total GPU usage on GPU device. Make sure to not have additional processes running"
													    " on the same GPU."						)
													# init nvml
													nvml.nvmlInit()
													func()
													__UpperCAmelCase     :				str					   =			nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx						)
													__UpperCAmelCase     :				Any					   =			nvml.nvmlDeviceGetMemoryInfo(lowerCAmelCase_						)
													__UpperCAmelCase     :				Dict					   =			meminfo.used
													__UpperCAmelCase     :				Optional[int]					   =			Memory(lowerCAmelCase_						)
													# shutdown nvml
													nvml.nvmlShutdown()
									else:
											# cpu
											__UpperCAmelCase     :				List[str]					   =			measure_peak_memory_cpu(lowerCAmelCase_						)
											__UpperCAmelCase     :				Tuple					   =			Memory(lowerCAmelCase_						) if isinstance(lowerCAmelCase_						,     lowerCAmelCase_						) else memory_bytes

									if self.args.trace_memory_line_by_line:
											__UpperCAmelCase     :				Optional[Any]					   =			stop_memory_tracing(lowerCAmelCase_						)
									else:
											__UpperCAmelCase     :				Union[str, Any]					   =			None

									return memory, summary
							except RuntimeError as e:
									self.print_fn(f"Doesn\'t fit on GPU. {e}"						)
									return "N/A", None










'''simple docstring'''


import os
from shutil import copyfile
from typing import Optional, Tuple

from ...tokenization_utils_fast import PreTrainedTokenizerFast
from ...utils import is_sentencepiece_available, logging


if is_sentencepiece_available():
				from .tokenization_reformer import ReformerTokenizer
else:
				_snake_case      :      int   							=		None


_snake_case      :      Union[str, Any]   							=		logging.get_logger(__name__)


_snake_case      :      Union[str, Any]   							=		'▁'

_snake_case      :      Any   							=		{'vocab_file': 'spiece.model', 'tokenizer_file': 'tokenizer.json'}

_snake_case      :      List[str]   							=		{
    'vocab_file': {
        'google/reformer-crime-and-punishment': (
            'https://huggingface.co/google/reformer-crime-and-punishment/resolve/main/spiece.model'
        )
    },
    'tokenizer_file': {
        'google/reformer-crime-and-punishment': (
            'https://huggingface.co/google/reformer-crime-and-punishment/resolve/main/tokenizer.json'
        )
    },
}

_snake_case      :      Union[str, Any]   							=		{
    'google/reformer-crime-and-punishment': 524288,
}

class 			A    (		_a    ):
							lowercase_							=				VOCAB_FILES_NAMES
							lowercase_							=				PRETRAINED_VOCAB_FILES_MAP
							lowercase_							=				PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
							lowercase_							=				['input_ids', 'attention_mask']
							lowercase_							=				ReformerTokenizer







							def __init__(  self       :  Any							,			lowerCAmelCase_       :  int=None							,			lowerCAmelCase_       :  Optional[Any]=None							,			lowerCAmelCase_       :  List[str]="</s>"							,			lowerCAmelCase_       :  Any="<unk>"							,			lowerCAmelCase_       :  List[str]=[]							,			**lowerCAmelCase_       :  List[Any]							,			)			->							Dict:


														"""simple docstring"""


														super().__init__(
														    lowerCAmelCase_							,			tokenizer_file=lowerCAmelCase_							,			eos_token=lowerCAmelCase_							,			unk_token=lowerCAmelCase_							,			additional_special_tokens=lowerCAmelCase_							,			**lowerCAmelCase_							,			)

														_a	     =				vocab_file
														_a	     =				False if not self.vocab_file else True







							def    __lowerCAmelCase			(  self       :  List[Any]							,			lowerCAmelCase_       :  str							,			lowerCAmelCase_       :  Optional[str] = None					)			->							Tuple[str]:


														"""simple docstring"""


														if not self.can_save_slow_tokenizer:
																					raise ValueError(
																					    '''Your fast tokenizer does not have the necessary information to save the vocabulary for a slow '''
																					    '''tokenizer.'''					)

														if not os.path.isdir(lowerCAmelCase_					):
																					logger.error(F'Vocabulary path ({save_directory}) should be a directory'					)
																					return
														_a	     =				os.path.join(
														    lowerCAmelCase_							,			(filename_prefix + '''-''' if filename_prefix else '''''') + VOCAB_FILES_NAMES['''vocab_file''']					)

														if os.path.abspath(self.vocab_file					) != os.path.abspath(lowerCAmelCase_					):
																					copyfile(self.vocab_file							,			lowerCAmelCase_					)

														return (out_vocab_file,)

