class_docstr: Benchmarks is a simple but feature-complete benchmarking script to compare memory and time performance of models in
  Transformers.
code: def run(self):
      result_dict = {
          model_name: {} for model_name in self.args.model_names
      }
      inference_result_time = copy.deepcopy(result_dict)
      inference_result_memory = copy.deepcopy(result_dict)
      train_result_time = copy.deepcopy(result_dict)
      train_result_memory = copy.deepcopy(result_dict)
  
      for c, model_name in enumerate(self.args.model_names):
          self.print_fn(f"{c + 1} / {len(self.args.model_names)}")
  
          model_dict = {
              "bs": self.args.batch_sizes,
              "ss": self.args.sequence_lengths,
              "result": {i: {} for i in self.args.batch_sizes},
          }
          inference_result_time[model_name] = copy.deepcopy(model_dict)
          inference_result_memory[model_name] = copy.deepcopy(
              model_dict
          )
          train_result_time[model_name] = copy.deepcopy(model_dict)
          train_result_memory[model_name] = copy.deepcopy(model_dict)
  
          inference_summary = train_summary = None
  
          for batch_size in self.args.batch_sizes:
              for sequence_length in self.args.sequence_lengths:
                  if self.args.inference:
                      if self.args.memory:
                          (
                              memory,
                              inference_summary,
                          ) = self.inference_memory(
                              model_name, batch_size, sequence_length
                          )
                          inference_result_memory[model_name]["result"][
                              batch_size
                          ][sequence_length] = memory
                      if self.args.speed:
                          time = self.inference_speed(
                              model_name, batch_size, sequence_length
                          )
                          inference_result_time[model_name]["result"][
                              batch_size
                          ][sequence_length] = time
  
                  if self.args.training:
                      if self.args.memory:
                          memory, train_summary = self.train_memory(
                              model_name, batch_size, sequence_length
                          )
                          train_result_memory[model_name]["result"][
                              batch_size
                          ][sequence_length] = memory
                      if self.args.speed:
                          time = self.train_speed(
                              model_name, batch_size, sequence_length
                          )
                          train_result_time[model_name]["result"][
                              batch_size
                          ][sequence_length] = time
  
      if self.args.inference:
          if self.args.speed:
              self.print_fn(
                  "\n"
                  + 20 * "="
                  + ("INFERENCE - SPEED - RESULT").center(40)
                  + 20 * "="
              )
              self.print_results(
                  inference_result_time, type_label="Time in s"
              )
              self.save_to_csv(
                  inference_result_time,
                  self.args.inference_time_csv_file,
              )
              if self.args.is_tpu:
                  self.print_fn(
                      "TPU was used for inference. Note that the time after compilation stabilized (after ~10"
                      " inferences model.forward(..) calls) was measured."
                  )
  
          if self.args.memory:
              self.print_fn(
                  "\n"
                  + 20 * "="
                  + ("INFERENCE - MEMORY - RESULT").center(40)
                  + 20 * "="
              )
              self.print_results(
                  inference_result_memory, type_label="Memory in MB"
              )
              self.save_to_csv(
                  inference_result_memory,
                  self.args.inference_memory_csv_file,
              )
  
          if self.args.trace_memory_line_by_line:
              self.print_fn(
                  "\n"
                  + 20 * "="
                  + (
                      "INFERENCE - MEMOMRY - LINE BY LINE - SUMMARY"
                  ).center(40)
                  + 20 * "="
              )
              self.print_memory_trace_statistics(inference_summary)
  
      if self.args.training:
          if self.args.speed:
              self.print_fn(
                  "\n"
                  + 20 * "="
                  + ("TRAIN - SPEED - RESULTS").center(40)
                  + 20 * "="
              )
              self.print_results(train_result_time, "Time in s")
              self.save_to_csv(
                  train_result_time, self.args.train_time_csv_file
              )
              if self.args.is_tpu:
                  self.print_fn(
                      "TPU was used for training. Note that the time after compilation stabilized (after ~10 train"
                      " loss=model.forward(...) + loss.backward() calls) was measured."
                  )
  
          if self.args.memory:
              self.print_fn(
                  "\n"
                  + 20 * "="
                  + ("TRAIN - MEMORY - RESULTS").center(40)
                  + 20 * "="
              )
              self.print_results(
                  train_result_memory, type_label="Memory in MB"
              )
              self.save_to_csv(
                  train_result_memory, self.args.train_memory_csv_file
              )
  
          if self.args.trace_memory_line_by_line:
              self.print_fn(
                  "\n"
                  + 20 * "="
                  + ("TRAIN - MEMOMRY - LINE BY LINE - SUMMARY").center(
                      40
                  )
                  + 20 * "="
              )
              self.print_memory_trace_statistics(train_summary)
  
      if self.args.env_print:
          self.print_fn(
              "\n"
              + 20 * "="
              + ("ENVIRONMENT INFORMATION").center(40)
              + 20 * "="
          )
          self.print_fn(
              "\n".join(
                  [
                      f"- {prop}: {val}"
                      for prop, val in self.environment_info.items()
                  ]
              )
              + "\n"
          )
  
      if self.args.save_to_csv:
          with open(
              self.args.env_info_csv_file, mode="w", newline=""
          ) as csv_file:
              writer = csv.writer(csv_file)
              for key, value in self.environment_info.items():
                  writer.writerow([key, value])
  
      return BenchmarkOutput(
          inference_result_time,
          inference_result_memory,
          train_result_time,
          train_result_memory,
          inference_summary,
          train_summary,
      )
  
imports: import copy
  import csv
  import linecache
  import os
  import platform
  import sys
  import warnings
  from abc import ABC, abstractmethod
  from collections import defaultdict, namedtuple
  from datetime import datetime
  from multiprocessing import Pipe, Process, Queue
  from multiprocessing.connection import Connection
  from typing import Callable, Iterable, List, NamedTuple, Optional, Union
  from ..None import AutoConfig, PretrainedConfig
  from ..None import __version__
  from ..utils import is_psutil_available, is_py3nvml_available, is_tf_available, is_torch_available, logging
  from .benchmark_args_utils import BenchmarkArguments
label_desc: Use image node_id 12 for calling the Benchmark obj's underlying member method code with example usage: obj.run() and returns: BenchmarkOutput