


'''simple docstring'''






import os
import tempfile
import unittest
from pathlib import Path

from transformers import AutoConfig, is_tf_available
from transformers.testing_utils import require_tf


if is_tf_available():
  import tensorflow as tf

  from transformers import TensorFlowBenchmark, TensorFlowBenchmarkArguments






@require_tf
class   __snake_case			(  unittest.TestCase):







    """simple docstring"""



    def __lowercase  (  self       :    List[str]       ,		lowerCamelCase       :    Optional[int]						)  ->    Any:
      for model_result in results.values():
        for batch_size, sequence_length in zip(model_result["""bs"""]       ,		model_result["""ss"""]						):
          lowerCAmelCase_     : Optional[Any]     =  model_result["""result"""][batch_size][sequence_length]
          self.assertIsNotNone(lowerCamelCase						)

    def __lowercase  (  self       :    List[str]						)  ->    List[Any]:
      lowerCAmelCase_     : int     =  """sshleifer/tiny-gpt2"""
      lowerCAmelCase_     : Union[str, Any]     =  TensorFlowBenchmarkArguments(
          models=[MODEL_ID]       ,		training=lowerCamelCase       ,		inference=lowerCamelCase       ,		sequence_lengths=[8]       ,		batch_sizes=[1]       ,		eager_mode=lowerCamelCase       ,		multi_process=lowerCamelCase       ,		)
      lowerCAmelCase_     : int     =  TensorFlowBenchmark(lowerCamelCase						)
      lowerCAmelCase_     : Dict     =  benchmark.run()
      self.check_results_dict_not_empty(results.time_inference_result						)
      self.check_results_dict_not_empty(results.memory_inference_result						)

    def __lowercase  (  self       :    Union[str, Any]						)  ->    Dict:
      lowerCAmelCase_     : Any     =  """sgugger/tiny-distilbert-classification"""
      lowerCAmelCase_     : List[Any]     =  TensorFlowBenchmarkArguments(
          models=[MODEL_ID]       ,		training=lowerCamelCase       ,		inference=lowerCamelCase       ,		sequence_lengths=[8]       ,		batch_sizes=[1]       ,		multi_process=lowerCamelCase       ,		only_pretrain_model=lowerCamelCase       ,		)
      lowerCAmelCase_     : Tuple     =  TensorFlowBenchmark(lowerCamelCase						)
      lowerCAmelCase_     : str     =  benchmark.run()
      self.check_results_dict_not_empty(results.time_inference_result						)
      self.check_results_dict_not_empty(results.memory_inference_result						)

    def __lowercase  (  self       :    List[Any]						)  ->    Optional[Any]:
      lowerCAmelCase_     : Any     =  """sshleifer/tiny-gpt2"""
      lowerCAmelCase_     : List[Any]     =  TensorFlowBenchmarkArguments(
          models=[MODEL_ID]       ,		training=lowerCamelCase       ,		inference=lowerCamelCase       ,		sequence_lengths=[8]       ,		batch_sizes=[1]       ,		multi_process=lowerCamelCase       ,		)
      lowerCAmelCase_     : int     =  TensorFlowBenchmark(lowerCamelCase						)
      lowerCAmelCase_     : Union[str, Any]     =  benchmark.run()
      self.check_results_dict_not_empty(results.time_inference_result						)
      self.check_results_dict_not_empty(results.memory_inference_result						)

    def __lowercase  (  self       :    List[Any]						)  ->    Dict:
      lowerCAmelCase_     : Union[str, Any]     =  """sshleifer/tiny-gpt2"""
      lowerCAmelCase_     : str     =  AutoConfig.from_pretrained(lowerCamelCase						)
      lowerCAmelCase_     : Any     =  TensorFlowBenchmarkArguments(
          models=[MODEL_ID]       ,		training=lowerCamelCase       ,		inference=lowerCamelCase       ,		sequence_lengths=[8]       ,		batch_sizes=[1]       ,		eager_mode=lowerCamelCase       ,		multi_process=lowerCamelCase       ,		)
      lowerCAmelCase_     : Dict     =  TensorFlowBenchmark(lowerCamelCase       ,		[config]						)
      lowerCAmelCase_     : int     =  benchmark.run()
      self.check_results_dict_not_empty(results.time_inference_result						)
      self.check_results_dict_not_empty(results.memory_inference_result						)

    def __lowercase  (  self       :    List[Any]						)  ->    Optional[Any]:
      lowerCAmelCase_     : str     =  """sshleifer/tiny-gpt2"""
      lowerCAmelCase_     : List[str]     =  AutoConfig.from_pretrained(lowerCamelCase						)
      lowerCAmelCase_     : Any     =  TensorFlowBenchmarkArguments(
          models=[MODEL_ID]       ,		training=lowerCamelCase       ,		inference=lowerCamelCase       ,		sequence_lengths=[8]       ,		batch_sizes=[1]       ,		multi_process=lowerCamelCase       ,		)
      lowerCAmelCase_     : Any     =  TensorFlowBenchmark(lowerCamelCase       ,		[config]						)
      lowerCAmelCase_     : List[str]     =  benchmark.run()
      self.check_results_dict_not_empty(results.time_inference_result						)
      self.check_results_dict_not_empty(results.memory_inference_result						)

    def __lowercase  (  self       :    Optional[Any]						)  ->    Union[str, Any]:
      lowerCAmelCase_     : Dict     =  """sshleifer/tiny-gpt2"""
      lowerCAmelCase_     : Union[str, Any]     =  TensorFlowBenchmarkArguments(
          models=[MODEL_ID]       ,		training=lowerCamelCase       ,		inference=lowerCamelCase       ,		sequence_lengths=[8]       ,		batch_sizes=[1]       ,		multi_process=lowerCamelCase       ,		)
      lowerCAmelCase_     : Union[str, Any]     =  TensorFlowBenchmark(lowerCamelCase						)
      lowerCAmelCase_     : Any     =  benchmark.run()
      self.check_results_dict_not_empty(results.time_train_result						)
      self.check_results_dict_not_empty(results.memory_train_result						)

    def __lowercase  (  self       :    Optional[int]						)  ->    Any:
      lowerCAmelCase_     : int     =  """sshleifer/tiny-gpt2"""
      lowerCAmelCase_     : Any     =  AutoConfig.from_pretrained(lowerCamelCase						)
      lowerCAmelCase_     : Dict     =  TensorFlowBenchmarkArguments(
          models=[MODEL_ID]       ,		training=lowerCamelCase       ,		inference=lowerCamelCase       ,		sequence_lengths=[8]       ,		batch_sizes=[1]       ,		multi_process=lowerCamelCase       ,		)
      lowerCAmelCase_     : int     =  TensorFlowBenchmark(lowerCamelCase       ,		[config]						)
      lowerCAmelCase_     : Optional[Any]     =  benchmark.run()
      self.check_results_dict_not_empty(results.time_train_result						)
      self.check_results_dict_not_empty(results.memory_train_result						)

    def __lowercase  (  self       :    Dict						)  ->    List[str]:
      lowerCAmelCase_     : str     =  """patrickvonplaten/t5-tiny-random"""
      lowerCAmelCase_     : Tuple     =  AutoConfig.from_pretrained(lowerCamelCase						)
      lowerCAmelCase_     : Union[str, Any]     =  TensorFlowBenchmarkArguments(
          models=[MODEL_ID]       ,		training=lowerCamelCase       ,		inference=lowerCamelCase       ,		sequence_lengths=[8]       ,		batch_sizes=[1]       ,		multi_process=lowerCamelCase       ,		)
      lowerCAmelCase_     : Optional[Any]     =  TensorFlowBenchmark(lowerCamelCase       ,		configs=[config]						)
      lowerCAmelCase_     : str     =  benchmark.run()
      self.check_results_dict_not_empty(results.time_inference_result						)
      self.check_results_dict_not_empty(results.memory_inference_result						)

    @unittest.skipIf(is_tf_available() and len(tf.config.list_physical_devices("""GPU"""						)						) == 0       ,		"""Cannot do xla on CPU."""						)
    def __lowercase  (  self       :    Tuple						)  ->    Optional[Any]:
      lowerCAmelCase_     : Optional[int]     =  """sshleifer/tiny-gpt2"""
      lowerCAmelCase_     : Tuple     =  TensorFlowBenchmarkArguments(
          models=[MODEL_ID]       ,		training=lowerCamelCase       ,		inference=lowerCamelCase       ,		sequence_lengths=[8]       ,		batch_sizes=[1]       ,		use_xla=lowerCamelCase       ,		multi_process=lowerCamelCase       ,		)
      lowerCAmelCase_     : Union[str, Any]     =  TensorFlowBenchmark(lowerCamelCase						)
      lowerCAmelCase_     : Tuple     =  benchmark.run()
      self.check_results_dict_not_empty(results.time_inference_result						)
      self.check_results_dict_not_empty(results.memory_inference_result						)

    def __lowercase  (  self       :    Tuple						)  ->    Dict:
      lowerCAmelCase_     : Dict     =  """sshleifer/tiny-gpt2"""
      with tempfile.TemporaryDirectory() as tmp_dir:
        lowerCAmelCase_     : str     =  TensorFlowBenchmarkArguments(
            models=[MODEL_ID]       ,		inference=lowerCamelCase       ,		save_to_csv=lowerCamelCase       ,		sequence_lengths=[8]       ,		batch_sizes=[1]       ,		inference_time_csv_file=os.path.join(lowerCamelCase       ,		"""inf_time.csv"""						)       ,		inference_memory_csv_file=os.path.join(lowerCamelCase       ,		"""inf_mem.csv"""						)       ,		env_info_csv_file=os.path.join(lowerCamelCase       ,		"""env.csv"""						)       ,		multi_process=lowerCamelCase       ,		)
        lowerCAmelCase_     : Any     =  TensorFlowBenchmark(lowerCamelCase						)
        benchmark.run()
        self.assertTrue(Path(os.path.join(lowerCamelCase       ,		"""inf_time.csv"""						)						).exists()						)
        self.assertTrue(Path(os.path.join(lowerCamelCase       ,		"""inf_mem.csv"""						)						).exists()						)
        self.assertTrue(Path(os.path.join(lowerCamelCase       ,		"""env.csv"""						)						).exists()						)





    def __lowercase  (  self       :    Any						)  ->    int:
      lowerCAmelCase_     : Optional[int]     =  """sshleifer/tiny-gpt2"""

      def _check_summary_is_not_empty(lowerCamelCase       :    List[str]						):
        self.assertTrue(hasattr(lowerCamelCase       ,		"""sequential"""						)						)
        self.assertTrue(hasattr(lowerCamelCase       ,		"""cumulative"""						)						)
        self.assertTrue(hasattr(lowerCamelCase       ,		"""current"""						)						)
        self.assertTrue(hasattr(lowerCamelCase       ,		"""total"""						)						)

      with tempfile.TemporaryDirectory() as tmp_dir:
        lowerCAmelCase_     : Optional[Any]     =  TensorFlowBenchmarkArguments(
            models=[MODEL_ID]       ,		inference=lowerCamelCase       ,		sequence_lengths=[8]       ,		batch_sizes=[1]       ,		log_filename=os.path.join(lowerCamelCase       ,		"""log.txt"""						)       ,		log_print=lowerCamelCase       ,		trace_memory_line_by_line=lowerCamelCase       ,		eager_mode=lowerCamelCase       ,		multi_process=lowerCamelCase       ,		)
        lowerCAmelCase_     : Optional[Any]     =  TensorFlowBenchmark(lowerCamelCase						)
        lowerCAmelCase_     : Tuple     =  benchmark.run()
        _check_summary_is_not_empty(result.inference_summary						)
        self.assertTrue(Path(os.path.join(lowerCamelCase       ,		"""log.txt"""						)						).exists()						)







'''simple docstring'''






import os

import pytest

from datasets import (
    get_dataset_config_info,
    get_dataset_config_names,
    get_dataset_infos,
    get_dataset_split_names,
    inspect_dataset,
    inspect_metric,
)


__A   :    int											= pytest.mark.integration







@pytest.mark.parametrize("""path"""				,       ["""paws""", """csv"""]   )
def 							UpperCamelCase_     (       A__   :  Union[str, Any]				,       A__   :  Tuple   ):
  '''simple docstring'''





  inspect_dataset(A__				,       A__   )
  lowerCAmelCase_     : Any     =  path + """.py"""
  assert script_name in os.listdir(A__   )
  assert "__pycache__" not in os.listdir(A__   )







@pytest.mark.filterwarnings("""ignore:inspect_metric is deprecated:FutureWarning"""   )
@pytest.mark.filterwarnings("""ignore:metric_module_factory is deprecated:FutureWarning"""   )
@pytest.mark.parametrize("""path"""				,       ["""accuracy"""]   )
def 							UpperCamelCase_     (       A__   :  Tuple				,       A__   :  str   ):
  '''simple docstring'''





  inspect_metric(A__				,       A__   )
  lowerCAmelCase_     : Any     =  path + """.py"""
  assert script_name in os.listdir(A__   )
  assert "__pycache__" not in os.listdir(A__   )







@pytest.mark.parametrize(
    """path, config_name, expected_splits"""				,       [
        ("""squad""", """plain_text""", ["""train""", """validation"""]),
        ("""dalle-mini/wit""", """dalle-mini--wit""", ["""train"""]),
        ("""paws""", """labeled_final""", ["""train""", """test""", """validation"""]),
    ]				,       )
def 							UpperCamelCase_     (       A__   :  List[str]				,       A__   :  List[str]				,       A__   :  Optional[Any]   ):
  '''simple docstring'''





  lowerCAmelCase_     : List[Any]     =  get_dataset_config_info(A__				,       config_name=A__   )
  assert info.config_name == config_name
  assert list(info.splits.keys()   ) == expected_splits







@pytest.mark.parametrize(
    """path, config_name, expected_exception"""				,       [
        ("""paws""", None, ValueError),
    ]				,       )
def 							UpperCamelCase_     (       A__   :  List[str]				,       A__   :  List[str]				,       A__   :  int   ):
  '''simple docstring'''





  with pytest.raises(A__   ):
    get_dataset_config_info(A__				,       config_name=A__   )







@pytest.mark.parametrize(
    """path, expected"""				,       [
        ("""squad""", """plain_text"""),
        ("""acronym_identification""", """default"""),
        ("""lhoestq/squad""", """plain_text"""),
        ("""lhoestq/test""", """default"""),
        ("""lhoestq/demo1""", """lhoestq--demo1"""),
        ("""dalle-mini/wit""", """dalle-mini--wit"""),
    ]				,       )
def 							UpperCamelCase_     (       A__   :  str				,       A__   :  List[str]   ):
  '''simple docstring'''





  lowerCAmelCase_     : Dict     =  get_dataset_config_names(A__   )
  assert expected in config_names







@pytest.mark.parametrize(
    """path, expected_configs, expected_splits_in_first_config"""				,       [
        ("""squad""", ["""plain_text"""], ["""train""", """validation"""]),
        ("""dalle-mini/wit""", ["""dalle-mini--wit"""], ["""train"""]),
        ("""paws""", ["""labeled_final""", """labeled_swap""", """unlabeled_final"""], ["""train""", """test""", """validation"""]),
    ]				,       )
def 							UpperCamelCase_     (       A__   :  List[Any]				,       A__   :  Dict				,       A__   :  Optional[int]   ):
  '''simple docstring'''





  lowerCAmelCase_     : Tuple     =  get_dataset_infos(A__   )
  assert list(infos.keys()   ) == expected_configs
  lowerCAmelCase_     : Optional[int]     =  expected_configs[0]
  assert expected_config in infos
  lowerCAmelCase_     : int     =  infos[expected_config]
  assert info.config_name == expected_config
  assert list(info.splits.keys()   ) == expected_splits_in_first_config







@pytest.mark.parametrize(
    """path, expected_config, expected_splits"""				,       [
        ("""squad""", """plain_text""", ["""train""", """validation"""]),
        ("""dalle-mini/wit""", """dalle-mini--wit""", ["""train"""]),
        ("""paws""", """labeled_final""", ["""train""", """test""", """validation"""]),
    ]				,       )
def 							UpperCamelCase_     (       A__   :  List[str]				,       A__   :  Optional[Any]				,       A__   :  List[Any]   ):
  '''simple docstring'''





  lowerCAmelCase_     : Dict     =  get_dataset_infos(A__   )
  assert expected_config in infos
  lowerCAmelCase_     : List[Any]     =  infos[expected_config]
  assert info.config_name == expected_config
  assert list(info.splits.keys()   ) == expected_splits







@pytest.mark.parametrize(
    """path, config_name, expected_exception"""				,       [
        ("""paws""", None, ValueError),
    ]				,       )
def 							UpperCamelCase_     (       A__   :  List[str]				,       A__   :  Any				,       A__   :  Dict   ):
  '''simple docstring'''





  with pytest.raises(A__   ):
    get_dataset_split_names(A__				,       config_name=A__   )



