






"""simple docstring"""


import timeit
from typing import Callable, Optional

from ..configuration_utils import PretrainedConfig
from ..models.auto.modeling_auto import MODEL_MAPPING, MODEL_WITH_LM_HEAD_MAPPING
from ..utils import is_pyanvml_available, is_torch_available, logging
from .benchmark_utils import (
    Benchmark,
    Memory,
    MemorySummary,
    measure_peak_memory_cpu,
    start_memory_tracing,
    stop_memory_tracing,
)


if is_torch_available():
   import torch

   from .benchmark_args import PyTorchBenchmarkArguments


if is_pyanvml_available():
   import pyanvml.pyanvml as nvml


_UpperCamelCase				:    Union[str, Any]     =						logging.get_logger(__name__)






class 						snake_case  (  UpperCAmelCase       ):
    __magic_name__         =     42
    __magic_name__         =     42
    __magic_name__         =     "PyTorch"





    @property
    def 				lowerCamelCase__      (							self	:					List[Any]			):


      '''simple docstring'''




      return torch.__version__





    def 				lowerCamelCase__      (							self	:					Union[str, Any]				,		A	:					str				,		A	:					int				,		A	:					int			):


      '''simple docstring'''




      a     :			str				=   self._prepare_inference_func(A				,		A				,		A			)
      return self._measure_speed(_inference			)





    def 				lowerCamelCase__      (							self	:					Optional[Any]				,		A	:					str				,		A	:					int				,		A	:					int			):


      '''simple docstring'''




      a     :			str				=   self._prepare_inference_func(A				,		A				,		A			)
      return self._measure_memory(_inference			)





    def 				lowerCamelCase__      (							self	:					Tuple				,		A	:					str				,		A	:					int				,		A	:					int			):


      '''simple docstring'''




      a     :			Optional[int]				=   self._prepare_train_func(A				,		A				,		A			)
      return self._measure_speed(_train			)





    def 				lowerCamelCase__      (							self	:					int				,		A	:					str				,		A	:					int				,		A	:					int			):


      '''simple docstring'''




      a     :			Any				=   self._prepare_train_func(A				,		A				,		A			)
      return self._measure_memory(_train			)





    def 				lowerCamelCase__      (							self	:					List[Any]				,		A	:					str				,		A	:					int				,		A	:					int			):


      '''simple docstring'''




      a     :			int				=   self.config_dict[model_name]

      if self.args.torchscript:
        a     :			Any				=   True

      a     :			Tuple				=   (
          hasattr(A				,		'architectures'			)
          and isinstance(config.architectures				,		A			)
          and len(config.architectures			) > 0
      )
      if not self.args.only_pretrain_model and has_model_class_in_config:
        try:
          a     :			Optional[int]				=   config.architectures[0]
          a     :			Optional[Any]				=   __import__('transformers'				,		fromlist=[model_class]			)
          a     :			Tuple				=   getattr(A				,		A			)
          a     :			int				=   model_cls(A			)
        except ImportError:
          raise ImportError(
              F'''{model_class} does not exist. If you just want to test the pretrained model, you might want to'''
              ' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'			)
      else:
        a     :			Dict				=   MODEL_MAPPING[config.__class__](A			)

      model.eval()
      model.to(self.args.device			)

      # encoder-decoder has vocab size saved differently
      a     :			Tuple				=   config.vocab_size if hasattr(A				,		'vocab_size'			) else config.encoder.vocab_size
      a     :			Tuple				=   torch.randint(A				,		(batch_size, sequence_length)				,		dtype=torch.long				,		device=self.args.device			)

      if self.args.fpaa:
        logger.info('Running training in Mixed Precision...'			)
        if not self.args.is_gpu:
          raise ValueError('Mixed precision is possible only for GPU.'			)
        # amp seems to have memory leaks so that memory usage
        # is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
        model.half()

      if self.args.torchscript:
        with torch.no_grad():
          a     :			List[str]				=   torch.jit.trace(A				,		A			)
      else:
        a     :			Optional[Any]				=   model

      def encoder_decoder_forward():
        with torch.no_grad():
          a     :			List[Any]				=   inference_model(A				,		decoder_input_ids=A			)
        return outputs

      def encoder_forward():
        with torch.no_grad():
          a     :			Optional[int]				=   inference_model(A			)
        return outputs

      a     :			Dict				=   encoder_decoder_forward if config.is_encoder_decoder else encoder_forward
      return _forward





    def 				lowerCamelCase__      (							self	:					Tuple				,		A	:					str				,		A	:					int				,		A	:					int			):


      '''simple docstring'''




      a     :			Optional[int]				=   self.config_dict[model_name]

      a     :			Optional[int]				=   (
          hasattr(A				,		'architectures'			)
          and isinstance(config.architectures				,		A			)
          and len(config.architectures			) > 0
      )
      if not self.args.only_pretrain_model and has_model_class_in_config:
        try:
          a     :			Optional[Any]				=   config.architectures[0]
          a     :			Dict				=   __import__('transformers'				,		fromlist=[model_class]			)
          a     :			List[str]				=   getattr(A				,		A			)
          a     :			str				=   model_cls(A			)
        except ImportError:
          raise ImportError(
              F'''{model_class} does not exist. If you just want to test the pretrained model, you might want to'''
              ' set `--only_pretrain_model` or `args.only_pretrain_model=True`.'			)
      else:
        a     :			Optional[Any]				=   MODEL_WITH_LM_HEAD_MAPPING[config.__class__](A			)

      if self.args.torchscript:
        raise NotImplementedError('Training for torchscript is currently not implemented'			)
      else:
        a     :			Optional[int]				=   model

      model.train()
      model.to(self.args.device			)

      # encoder-decoder has vocab size saved differently
      a     :			Any				=   config.vocab_size if hasattr(A				,		'vocab_size'			) else config.encoder.vocab_size
      a     :			Optional[Any]				=   torch.randint(A				,		(batch_size, sequence_length)				,		dtype=torch.long				,		device=self.args.device			)

      if self.args.fpaa:
        logger.info('Running training in Mixed Precision...'			)
        if not self.args.is_gpu:
          raise ValueError('Mixed precision is possible only for GPU.'			)

        # amp seems to have memory leaks so that memory usage
        # is measured using .half() for now https://github.com/NVIDIA/apex/issues/439
        model.half()

      def compute_loss_and_backprob_encoder():
        a     :			str				=   train_model(A				,		labels=A			)[0]
        loss.backward()
        return loss

      def compute_loss_and_backprob_encoder_decoder():
        a     :			Optional[Any]				=   train_model(A				,		decoder_input_ids=A				,		labels=A			)[0]
        loss.backward()
        return loss

      a     :			Dict				=   (
          compute_loss_and_backprob_encoder_decoder
          if config.is_encoder_decoder
          else compute_loss_and_backprob_encoder
      )
      return _train





    def 				lowerCamelCase__      (							self	:					Optional[int]				,		A	:					Optional[int]			):


      '''simple docstring'''




      try:
        if self.args.is_tpu or self.args.torchscript:
          # run additional 10 times to stabilize compilation for tpu and torchscript
          logger.info('Do inference on TPU or torchscript. Running model 5 times to stabilize compilation'			)
          timeit.repeat(
              A				,		repeat=1				,		number=5				,		)

        # as written in https://docs.python.org/2/library/timeit.html#timeit.Timer.repeat, min should be taken rather than the average
        a     :			List[Any]				=   timeit.repeat(
            A				,		repeat=self.args.repeat				,		number=1_0				,		)

        if self.args.is_tpu and self.args.torch_xla_tpu_print_metrics:
          import torch_xla.debug.metrics as met

          self.print_fn(met.metrics_report()			)

        return min(A			) / 10.0
      except RuntimeError as e:
        self.print_fn(F'''Doesn\'t fit on GPU. {e}'''			)
        return "N/A"





    def 				lowerCamelCase__      (							self	:					List[str]				,		A	:					Callable[[], None]			):


      '''simple docstring'''




      try:
        if self.args.trace_memory_line_by_line:
          a     :			int				=   start_memory_tracing('transformers'			)

        if self.args.is_tpu:
          # tpu
          raise NotImplementedError(
              'Memory Benchmarking is currently not implemented for TPU. Please disable memory benchmarking with'
              ' `--no-memory` or `args.memory=False`'			)
        elif self.args.is_gpu:
          if not is_pyanvml_available():
            logger.warning(
                'py3nvml not installed, we won\'t log GPU memory usage. '
                'Install py3nvml (pip install py3nvml) to log information about GPU.'			)
            a     :			List[Any]				=   'N/A'
          else:
            logger.info(
                'Measuring total GPU usage on GPU device. Make sure to not have additional processes running'
                ' on the same GPU.'			)
            # init nvml
            nvml.nvmlInit()
            func()
            a     :			List[str]				=   nvml.nvmlDeviceGetHandleByIndex(self.args.device_idx			)
            a     :			Optional[int]				=   nvml.nvmlDeviceGetMemoryInfo(A			)
            a     :			Tuple				=   meminfo.used
            a     :			Any				=   Memory(A			)
            # shutdown nvml
            nvml.nvmlShutdown()
        else:
          # cpu
          a     :			str				=   measure_peak_memory_cpu(A			)
          a     :			Tuple				=   Memory(A			) if isinstance(A				,		A			) else memory_bytes

        if self.args.trace_memory_line_by_line:
          a     :			str				=   stop_memory_tracing(A			)
        else:
          a     :			List[Any]				=   None

        return memory, summary
      except RuntimeError as e:
        self.print_fn(F'''Doesn\'t fit on GPU. {e}'''			)
        return "N/A", None





import json
import os
from dataclasses import dataclass

import numpy as np
import pyarrow as pa

import datasets
from utils import get_duration


_lowerCamelCase    :      int	 =						10_00_00_00_00_00
_lowerCamelCase    :      Union[str, Any]	 =						1_00_00

_lowerCamelCase ,  _lowerCamelCase    :      List[str]	 =						os.path.split(__file__)
_lowerCamelCase    :      Any	 =						os.path.join(RESULTS_BASEPATH, '''results''', RESULTS_FILENAME.replace('''.py''', '''.json'''))





def 		__lowerCamelCase							(UpperCAmelCase__   :							int					,       UpperCAmelCase__   :							int     ):
					SCREAMING_SNAKE_CASE				=				pa.Table.from_pydict({"col": [0] * chunk_size}     )
					SCREAMING_SNAKE_CASE				=				pa.concat_tables([table] * (num_examples // chunk_size)     )
					return datasets.Dataset(UpperCAmelCase__					,       fingerprint="table_100B"     )


@dataclass
class   lowercase			:
							lowercase__  :				int
							lowercase__  :				int
							lowercase__  :				int
							lowercase__  :				int




							def 		__snake_case(  self      :    Tuple					)    -> Dict:



												'''simple docstring'''

												SCREAMING_SNAKE_CASE				=				np.random.default_rng(self.seed					)
												SCREAMING_SNAKE_CASE				=				rng.integers(low=self.low		,							high=self.high		,							size=self.size					).tolist()




							def __iter__(  self      :    Union[str, Any]					)    -> Dict:



												'''simple docstring'''

												return iter(self._sampled_values					)





							def __len__(  self      :    Any					)    -> int:



												'''simple docstring'''

												return self.size





@get_duration
def 		__lowerCamelCase							(UpperCAmelCase__   :							datasets.Dataset     ):
					SCREAMING_SNAKE_CASE				=				dataset[0]





@get_duration
def 		__lowerCamelCase							(UpperCAmelCase__   :							datasets.Dataset     ):
					SCREAMING_SNAKE_CASE				=				dataset[-1]





@get_duration
def 		__lowerCamelCase							(UpperCAmelCase__   :							datasets.Dataset     ):
					SCREAMING_SNAKE_CASE				=				dataset[range(len(UpperCAmelCase__     ) // 2					,       len(UpperCAmelCase__     ) // 2 + 1_0_2_4     )]





@get_duration
def 		__lowerCamelCase							(UpperCAmelCase__   :							datasets.Dataset     ):
					SCREAMING_SNAKE_CASE				=				dataset[RandIter(0					,       len(UpperCAmelCase__     )					,       1_0_2_4					,       seed=4_2     )]





def 		__lowerCamelCase							():
					SCREAMING_SNAKE_CASE				=				{"num examples": SPEED_TEST_N_EXAMPLES}
					SCREAMING_SNAKE_CASE				=				(get_first_row, get_last_row, get_batch_of_aaaa_rows, get_batch_of_aaaa_random_rows)
					print("generating dataset"     )
					SCREAMING_SNAKE_CASE				=				generate_100B_dataset(num_examples=UpperCAmelCase__					,       chunk_size=UpperCAmelCase__     )
					print("Functions"     )
					for func in functions:
										print(func.__name__     )
										SCREAMING_SNAKE_CASE				=				func(UpperCAmelCase__     )

					with open(UpperCAmelCase__					,       "wb"     ) as f:
										f.write(json.dumps(UpperCAmelCase__     ).encode("utf-8"     )     )


if __name__ == "__main__":  # useful to run the profiler
							benchmark_table_100B()




