


import contextlib
import importlib
import io
import unittest

import transformers

# Try to import everything from transformers to ensure every object can be loaded.
from transformers import *  # noqa F406
from transformers.testing_utils import DUMMY_UNKNOWN_IDENTIFIER, require_flax, require_tf, require_torch
from transformers.utils import ContextManagers, find_labels, is_flax_available, is_tf_available, is_torch_available


if is_torch_available():
   from transformers import BertForPreTraining, BertForQuestionAnswering, BertForSequenceClassification

if is_tf_available():
   from transformers import TFBertForPreTraining, TFBertForQuestionAnswering, TFBertForSequenceClassification

if is_flax_available():
   from transformers import FlaxBertForPreTraining, FlaxBertForQuestionAnswering, FlaxBertForSequenceClassification


SCREAMING_SNAKE_CASE						:List[str]			=  DUMMY_UNKNOWN_IDENTIFIER
# An actual model hosted on huggingface.co

SCREAMING_SNAKE_CASE						:List[Any]			=  '''main'''
# Default branch name
SCREAMING_SNAKE_CASE						:int			=  '''f2c752cfc5c0ab6f4bdec59acea69eefbee381c2'''
# One particular commit (not the top of `main`)
SCREAMING_SNAKE_CASE						:str			=  '''aaaaaaa'''
# This commit does not exist, so we should 404.

SCREAMING_SNAKE_CASE						:Dict			=  '''d9e9f15bc825e4b2c9249e9578f884bbcb5e3684'''
# Sha-1 of config.json on the top of `main`, for checking purposes
SCREAMING_SNAKE_CASE						:Union[str, Any]			=  '''4b243c475af8d0a7754e87d7d096c92e5199ec2fe168a2ee7998e3b8e9bcb1d3'''



@contextlib.contextmanager
def       _lowerCAmelCase    (				)->Union[str, Any]:
   '''simple docstring'''






   print("Welcome!"				)
   yield
   print("Bye!"				)



@contextlib.contextmanager
def       _lowerCAmelCase    (				)->List[str]:
   '''simple docstring'''






   print("Bonjour!"				)
   yield
   print("Au revoir!"				)


class      __lowerCAmelCase  (			unittest.TestCase       ):






  """simple docstring"""





  def        lowerCAmelCase__	(  self							:							Optional[Any]	) ->			Optional[int]:




     """simple docstring"""
     # If the spec is missing, importlib would not be able to import the module dynamically.
     assert transformers.__spec__ is not None
     assert importlib.util.find_spec("transformers"	) is not None






class      __lowerCAmelCase  (			unittest.TestCase       ):






  """simple docstring"""





  @unittest.mock.patch("sys.stdout"     ,					new_callable=io.StringIO	)
  def        lowerCAmelCase__	(  self							:							Dict     ,					_lowerCAmelCase							:							Dict	) ->			Optional[Any]:




     """simple docstring"""
     with ContextManagers([]	):
        print("Transformers are awesome!"	)
     # The print statement adds a new line at the end of the output
     self.assertEqual(mock_stdout.getvalue()     ,					"Transformers are awesome!\n"	)



  @unittest.mock.patch("sys.stdout"     ,					new_callable=io.StringIO	)
  def        lowerCAmelCase__	(  self							:							Union[str, Any]     ,					_lowerCAmelCase							:							Union[str, Any]	) ->			Optional[Any]:




     """simple docstring"""
     with ContextManagers([context_en()]	):
        print("Transformers are awesome!"	)
     # The output should be wrapped with an English welcome and goodbye
     self.assertEqual(mock_stdout.getvalue()     ,					"Welcome!\nTransformers are awesome!\nBye!\n"	)



  @unittest.mock.patch("sys.stdout"     ,					new_callable=io.StringIO	)
  def        lowerCAmelCase__	(  self							:							List[str]     ,					_lowerCAmelCase							:							Optional[Any]	) ->			Optional[Any]:




     """simple docstring"""
     with ContextManagers([context_fr(), context_en()]	):
        print("Transformers are awesome!"	)
     # The output should be wrapped with an English and French welcome and goodbye
     self.assertEqual(mock_stdout.getvalue()     ,					"Bonjour!\nWelcome!\nTransformers are awesome!\nBye!\nAu revoir!\n"	)



  @require_torch
  def        lowerCAmelCase__	(  self							:							Union[str, Any]	) ->			List[str]:




     """simple docstring"""
     self.assertEqual(find_labels(_lowerCAmelCase	)     ,					["labels"]	)
     self.assertEqual(find_labels(_lowerCAmelCase	)     ,					["labels", "next_sentence_label"]	)
     self.assertEqual(find_labels(_lowerCAmelCase	)     ,					["start_positions", "end_positions"]	)


     class      __lowerCAmelCase  (			a       ):






       """simple docstring"""





       pass






     self.assertEqual(find_labels(_lowerCAmelCase	)     ,					["labels"]	)



  @require_tf
  def        lowerCAmelCase__	(  self							:							Dict	) ->			Tuple:




     """simple docstring"""
     self.assertEqual(find_labels(_lowerCAmelCase	)     ,					["labels"]	)
     self.assertEqual(find_labels(_lowerCAmelCase	)     ,					["labels", "next_sentence_label"]	)
     self.assertEqual(find_labels(_lowerCAmelCase	)     ,					["start_positions", "end_positions"]	)


     class      __lowerCAmelCase  (			a       ):






       """simple docstring"""





       pass






     self.assertEqual(find_labels(_lowerCAmelCase	)     ,					["labels"]	)



  @require_flax
  def        lowerCAmelCase__	(  self							:							Any	) ->			List[str]:




     """simple docstring"""
     # Flax models don't have labels
     self.assertEqual(find_labels(_lowerCAmelCase	)     ,					[]	)
     self.assertEqual(find_labels(_lowerCAmelCase	)     ,					[]	)
     self.assertEqual(find_labels(_lowerCAmelCase	)     ,					[]	)


     class      __lowerCAmelCase  (			a       ):






       """simple docstring"""





       pass






     self.assertEqual(find_labels(_lowerCAmelCase	)     ,					[]	)








import math
import time
from typing import Dict, List, Optional

from torch.utils.data import Dataset

from transformers import SeqaSeqTrainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput, speed_metrics


if is_torch_tpu_available(check_device=False):
   import torch_xla.core.xla_model as xm
   import torch_xla.debug.metrics as met


class      __lowerCAmelCase  (			a       ):






  """simple docstring"""





  def __init__(  self							:							Union[str, Any]     ,					*_lowerCAmelCase							:							Dict     ,					_lowerCAmelCase							:							List[Any]=None     ,					_lowerCAmelCase							:							str=None     ,					**_lowerCAmelCase							:							str	) ->			Dict:




     """simple docstring"""
     super().__init__(*_lowerCAmelCase     ,					**_lowerCAmelCase	)
     snake_case_            =       eval_examples
     snake_case_            =       post_process_function



  def        lowerCAmelCase__	(  self							:							Union[str, Any]     ,					_lowerCAmelCase							:							Optional[Dataset] = None     ,					_lowerCAmelCase							:							Any=None     ,					_lowerCAmelCase							:							Optional[List[str]] = None     ,					_lowerCAmelCase							:							str = "eval"     ,					**_lowerCAmelCase							:							Tuple     ,					) ->			Dict[str, float]:




     """simple docstring"""
     snake_case_            =       gen_kwargs.copy()
     snake_case_            =       (
         gen_kwargs["max_length"] if gen_kwargs.get("max_length"	) is not None else self.args.generation_max_length
     )
     snake_case_            =       (
         gen_kwargs["num_beams"] if gen_kwargs.get("num_beams"	) is not None else self.args.generation_num_beams
     )
     snake_case_            =       gen_kwargs

     snake_case_            =       self.eval_dataset if eval_dataset is None else eval_dataset
     snake_case_            =       self.get_eval_dataloader(_lowerCAmelCase	)
     snake_case_            =       self.eval_examples if eval_examples is None else eval_examples

     # Temporarily disable metric computation, we will do it in the loop here.
     snake_case_            =       self.compute_metrics
     snake_case_            =       None
     snake_case_            =       time.time()
     snake_case_            =       self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
     try:
        snake_case_            =       eval_loop(
            _lowerCAmelCase     ,					description="Evaluation"     ,					prediction_loss_only=True if compute_metrics is None else None     ,					ignore_keys=_lowerCAmelCase     ,					metric_key_prefix=_lowerCAmelCase     ,					)
     finally:
        snake_case_            =       compute_metrics
     snake_case_            =       self.args.eval_batch_size * self.args.world_size
     if F'''{metric_key_prefix}_jit_compilation_time''' in output.metrics:
        start_time += output.metrics[F'''{metric_key_prefix}_jit_compilation_time''']
     output.metrics.update(
         speed_metrics(
             _lowerCAmelCase     ,					_lowerCAmelCase     ,					num_samples=output.num_samples     ,					num_steps=math.ceil(output.num_samples / total_batch_size	)     ,					)	)

     if self.post_process_function is not None and self.compute_metrics is not None and self.args.should_save:
        # Only the main node write the results by default
        snake_case_            =       self.post_process_function(_lowerCAmelCase     ,					_lowerCAmelCase     ,					_lowerCAmelCase	)
        snake_case_            =       self.compute_metrics(_lowerCAmelCase	)

        # Prefix all keys with metric_key_prefix + '_'
        for key in list(metrics.keys()	):
           if not key.startswith(F'''{metric_key_prefix}_'''	):
              snake_case_            =       metrics.pop(_lowerCAmelCase	)

        metrics.update(output.metrics	)
     else:
        snake_case_            =       output.metrics

     if self.args.should_log:
        # Only the main node log the results by default
        self.log(_lowerCAmelCase	)

     if self.args.tpu_metrics_debug or self.args.debug:
        # tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
        xm.master_print(met.metrics_report()	)

     snake_case_            =       self.callback_handler.on_evaluate(self.args     ,					self.state     ,					self.control     ,					_lowerCAmelCase	)
     return metrics



  def        lowerCAmelCase__	(  self							:							Optional[Any]     ,					_lowerCAmelCase							:							Optional[Any]     ,					_lowerCAmelCase							:							Union[str, Any]     ,					_lowerCAmelCase							:							List[str]=None     ,					_lowerCAmelCase							:							str = "test"     ,					**_lowerCAmelCase							:							Optional[Any]	) ->			Union[str, Any]:




     """simple docstring"""
     snake_case_            =       gen_kwargs.copy()

     snake_case_            =       self.get_test_dataloader(_lowerCAmelCase	)

     # Temporarily disable metric computation, we will do it in the loop here.
     snake_case_            =       self.compute_metrics
     snake_case_            =       None
     snake_case_            =       time.time()
     snake_case_            =       self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
     try:
        snake_case_            =       eval_loop(
            _lowerCAmelCase     ,					description="Prediction"     ,					prediction_loss_only=True if compute_metrics is None else None     ,					ignore_keys=_lowerCAmelCase     ,					metric_key_prefix=_lowerCAmelCase     ,					)
     finally:
        snake_case_            =       compute_metrics

     snake_case_            =       self.args.eval_batch_size * self.args.world_size
     if F'''{metric_key_prefix}_jit_compilation_time''' in output.metrics:
        start_time += output.metrics[F'''{metric_key_prefix}_jit_compilation_time''']
     output.metrics.update(
         speed_metrics(
             _lowerCAmelCase     ,					_lowerCAmelCase     ,					num_samples=output.num_samples     ,					num_steps=math.ceil(output.num_samples / total_batch_size	)     ,					)	)
     if self.post_process_function is None or self.compute_metrics is None:
        return output

     snake_case_            =       self.post_process_function(_lowerCAmelCase     ,					_lowerCAmelCase     ,					_lowerCAmelCase     ,					"predict"	)
     snake_case_            =       self.compute_metrics(_lowerCAmelCase	)

     # Prefix all keys with metric_key_prefix + '_'
     for key in list(metrics.keys()	):
        if not key.startswith(F'''{metric_key_prefix}_'''	):
           snake_case_            =       metrics.pop(_lowerCAmelCase	)
     metrics.update(output.metrics	)
     return PredictionOutput(predictions=predictions.predictions     ,					label_ids=predictions.label_ids     ,					metrics=_lowerCAmelCase	)




