





"""simple docstring"""





import math
import time
from typing import Dict, List, Optional

from torch.utils.data import Dataset

from transformers import SeqaSeqTrainer, is_torch_tpu_available
from transformers.trainer_utils import PredictionOutput, speed_metrics


if is_torch_tpu_available(check_device=False):
  import torch_xla.core.xla_model as xm
  import torch_xla.debug.metrics as met




class 							snake_case							(			__UpperCAmelCase ):

      '''simple docstring'''



      def __init__(							self       :      Tuple						,				*__lowercase       :      Any						,				__lowercase       :      Optional[Any]=None						,				__lowercase       :      Tuple=None						,				**__lowercase       :      Any ):


        '''simple docstring'''




        super().__init__(*__lowercase						,				**__lowercase )
        __UpperCAmelCase    :						Tuple       =						eval_examples
        __UpperCAmelCase    :						Dict       =						post_process_function

      def       A_					(							self       :      Union[str, Any]						,				__lowercase       :      Optional[Dataset] = None						,				__lowercase       :      str=None						,				__lowercase       :      Optional[List[str]] = None						,				__lowercase       :      str = "eval"						,				**__lowercase       :      Any						,				):


        '''simple docstring'''




        __UpperCAmelCase    :						List[str]       =						gen_kwargs.copy()
        __UpperCAmelCase    :						int       =						(
            gen_kwargs['''max_length'''] if gen_kwargs.get('''max_length''' ) is not None else self.args.generation_max_length
        )
        __UpperCAmelCase    :						Dict       =						(
            gen_kwargs['''num_beams'''] if gen_kwargs.get('''num_beams''' ) is not None else self.args.generation_num_beams
        )
        __UpperCAmelCase    :						Dict       =						gen_kwargs

        __UpperCAmelCase    :						List[str]       =						self.eval_dataset if eval_dataset is None else eval_dataset
        __UpperCAmelCase    :						str       =						self.get_eval_dataloader(__lowercase )
        __UpperCAmelCase    :						Union[str, Any]       =						self.eval_examples if eval_examples is None else eval_examples

        # Temporarily disable metric computation, we will do it in the loop here.
        __UpperCAmelCase    :						List[str]       =						self.compute_metrics
        __UpperCAmelCase    :						Tuple       =						None
        __UpperCAmelCase    :						str       =						time.time()
        __UpperCAmelCase    :						Optional[int]       =						self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
        try:
          __UpperCAmelCase    :						Union[str, Any]       =						eval_loop(
              __lowercase						,				description='''Evaluation'''						,				prediction_loss_only=True if compute_metrics is None else None						,				ignore_keys=__lowercase						,				metric_key_prefix=__lowercase						,				)
        finally:
          __UpperCAmelCase    :						Any       =						compute_metrics
        __UpperCAmelCase    :						Tuple       =						self.args.eval_batch_size * self.args.world_size
        if f'''{metric_key_prefix}_jit_compilation_time''' in output.metrics:
          start_time += output.metrics[f'''{metric_key_prefix}_jit_compilation_time''']
        output.metrics.update(
            speed_metrics(
                __lowercase						,				__lowercase						,				num_samples=output.num_samples						,				num_steps=math.ceil(output.num_samples / total_batch_size )						,				) )

        if self.post_process_function is not None and self.compute_metrics is not None and self.args.should_save:
          # Only the main node write the results by default
          __UpperCAmelCase    :						str       =						self.post_process_function(__lowercase						,				__lowercase						,				__lowercase )
          __UpperCAmelCase    :						Union[str, Any]       =						self.compute_metrics(__lowercase )

          # Prefix all keys with metric_key_prefix + '_'
          for key in list(metrics.keys() ):
            if not key.startswith(f'''{metric_key_prefix}_''' ):
              __UpperCAmelCase    :						List[Any]       =						metrics.pop(__lowercase )

          metrics.update(output.metrics )
        else:
          __UpperCAmelCase    :						Optional[int]       =						output.metrics

        if self.args.should_log:
          # Only the main node log the results by default
          self.log(__lowercase )

        if self.args.tpu_metrics_debug or self.args.debug:
          # tpu-comment: Logging debug metrics for PyTorch/XLA (compile, execute times, ops, etc.)
          xm.master_print(met.metrics_report() )

        __UpperCAmelCase    :						str       =						self.callback_handler.on_evaluate(self.args						,				self.state						,				self.control						,				__lowercase )
        return metrics







      def       A_					(							self       :      Union[str, Any]						,				__lowercase       :      Tuple						,				__lowercase       :      Dict						,				__lowercase       :      List[Any]=None						,				__lowercase       :      str = "test"						,				**__lowercase       :      Optional[int] ):


        '''simple docstring'''




        __UpperCAmelCase    :						Union[str, Any]       =						gen_kwargs.copy()

        __UpperCAmelCase    :						Optional[int]       =						self.get_test_dataloader(__lowercase )

        # Temporarily disable metric computation, we will do it in the loop here.
        __UpperCAmelCase    :						Union[str, Any]       =						self.compute_metrics
        __UpperCAmelCase    :						List[Any]       =						None
        __UpperCAmelCase    :						Optional[int]       =						time.time()
        __UpperCAmelCase    :						List[Any]       =						self.prediction_loop if self.args.use_legacy_prediction_loop else self.evaluation_loop
        try:
          __UpperCAmelCase    :						List[str]       =						eval_loop(
              __lowercase						,				description='''Prediction'''						,				prediction_loss_only=True if compute_metrics is None else None						,				ignore_keys=__lowercase						,				metric_key_prefix=__lowercase						,				)
        finally:
          __UpperCAmelCase    :						Dict       =						compute_metrics

        __UpperCAmelCase    :						List[str]       =						self.args.eval_batch_size * self.args.world_size
        if f'''{metric_key_prefix}_jit_compilation_time''' in output.metrics:
          start_time += output.metrics[f'''{metric_key_prefix}_jit_compilation_time''']
        output.metrics.update(
            speed_metrics(
                __lowercase						,				__lowercase						,				num_samples=output.num_samples						,				num_steps=math.ceil(output.num_samples / total_batch_size )						,				) )
        if self.post_process_function is None or self.compute_metrics is None:
          return output

        __UpperCAmelCase    :						List[Any]       =						self.post_process_function(__lowercase						,				__lowercase						,				__lowercase						,				'''predict''' )
        __UpperCAmelCase    :						Optional[Any]       =						self.compute_metrics(__lowercase )

        # Prefix all keys with metric_key_prefix + '_'
        for key in list(metrics.keys() ):
          if not key.startswith(f'''{metric_key_prefix}_''' ):
            __UpperCAmelCase    :						Optional[Any]       =						metrics.pop(__lowercase )
        metrics.update(output.metrics )
        return PredictionOutput(predictions=predictions.predictions						,				label_ids=predictions.label_ids						,				metrics=__lowercase )






"""simple docstring"""





from ...configuration_utils import PretrainedConfig
from ...utils import logging


lowercase__				:int				   =       logging.get_logger(__name__)

lowercase__				:List[str]				   =       {
    'EleutherAI/gpt-neox-20b': 'https://huggingface.co/EleutherAI/gpt-neox-20b/resolve/main/config.json',
    # See all GPTNeoX models at https://huggingface.co/models?filter=gpt_neox
}




class 							snake_case							(			__UpperCAmelCase ):

      '''simple docstring'''



      _A     :       List[Any]         =       'gpt_neox'

      def __init__(							self       :      Union[str, Any]						,				__lowercase       :      Any=50_432						,				__lowercase       :      int=6_144						,				__lowercase       :      List[str]=44						,				__lowercase       :      List[Any]=64						,				__lowercase       :      int=24_576						,				__lowercase       :      List[str]="gelu"						,				__lowercase       :      Optional[int]=0.2_5						,				__lowercase       :      Any=10_000						,				__lowercase       :      Union[str, Any]=0.0						,				__lowercase       :      List[str]=0.0						,				__lowercase       :      Union[str, Any]=0.1						,				__lowercase       :      Optional[int]=2_048						,				__lowercase       :      Union[str, Any]=0.0_2						,				__lowercase       :      Optional[int]=1e-5						,				__lowercase       :      str=True						,				__lowercase       :      str=0						,				__lowercase       :      Tuple=2						,				__lowercase       :      Tuple=False						,				__lowercase       :      int=True						,				__lowercase       :      Any=None						,				**__lowercase       :      List[str]						,				):


        '''simple docstring'''




        super().__init__(bos_token_id=__lowercase						,				eos_token_id=__lowercase						,				**__lowercase )
        __UpperCAmelCase    :						int       =						vocab_size
        __UpperCAmelCase    :						Union[str, Any]       =						max_position_embeddings
        __UpperCAmelCase    :						List[str]       =						hidden_size
        __UpperCAmelCase    :						Tuple       =						num_hidden_layers
        __UpperCAmelCase    :						Any       =						num_attention_heads
        __UpperCAmelCase    :						Union[str, Any]       =						intermediate_size
        __UpperCAmelCase    :						str       =						hidden_act
        __UpperCAmelCase    :						str       =						rotary_pct
        __UpperCAmelCase    :						str       =						rotary_emb_base
        __UpperCAmelCase    :						Dict       =						attention_dropout
        __UpperCAmelCase    :						str       =						hidden_dropout
        __UpperCAmelCase    :						Any       =						classifier_dropout
        __UpperCAmelCase    :						Tuple       =						initializer_range
        __UpperCAmelCase    :						Any       =						layer_norm_eps
        __UpperCAmelCase    :						Optional[int]       =						use_cache
        __UpperCAmelCase    :						List[Any]       =						tie_word_embeddings
        __UpperCAmelCase    :						Any       =						use_parallel_residual
        __UpperCAmelCase    :						List[str]       =						rope_scaling
        self._rope_scaling_validation()

        if self.hidden_size % self.num_attention_heads != 0:
          raise ValueError(
              '''The hidden size is not divisble by the number of attention heads! Make sure to update them!''' )







      def       A_					(							self       :      int ):


        '''simple docstring'''




        if self.rope_scaling is None:
          return

        if not isinstance(self.rope_scaling						,				__lowercase ) or len(self.rope_scaling ) != 2:
          raise ValueError(
              '''`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, '''
              f'''got {self.rope_scaling}''' )
        __UpperCAmelCase    :						Union[str, Any]       =						self.rope_scaling.get('''type'''						,				__lowercase )
        __UpperCAmelCase    :						List[Any]       =						self.rope_scaling.get('''factor'''						,				__lowercase )
        if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
          raise ValueError(
              f'''`rope_scaling`\'s name field must be one of [\'linear\', \'dynamic\'], got {rope_scaling_type}''' )
        if rope_scaling_factor is None or not isinstance(__lowercase						,				__lowercase ) or rope_scaling_factor <= 1.0:
          raise ValueError(f'''`rope_scaling`\'s factor field must be an float > 1, got {rope_scaling_factor}''' )