'''simple docstring'''






import re
import time
from typing import Optional

import IPython.display as disp

from ..trainer_callback import TrainerCallback
from ..trainer_utils import IntervalStrategy, has_length





def  _SCREAMING_SNAKE_CASE		(      __snake_case							:			str				):
     _A    		=  int(A__				)
     _A		,       _A		,       _A    		=  t // 3_6_0_0, (t // 6_0) % 6_0, t % 6_0
     return F'{h}:{m:02d}:{s:02d}' if h != 0 else F'{m:02d}:{s:02d}'





def  _SCREAMING_SNAKE_CASE		(      __snake_case							:			Dict   ,		__snake_case							:			Union[str, Any]   ,		__snake_case							:			Dict   ,		__snake_case							:			Tuple   ,		__snake_case							:			Union[str, Any]=3_0_0				):
     # docstyle-ignore
     return F'\n    <div>\n      {prefix}\n      <progress value=\'{value}\' max=\'{total}\' style=\'width:{width}px; height:20px; vertical-align: middle;\'></progress>\n      {label}\n    </div>\n    '





def  _SCREAMING_SNAKE_CASE		(      __snake_case							:			str				):
     _A    		=  '<table border=\"1\" class=\"dataframe\">\n'
     html_code += """  <thead>\n <tr style="text-align: left;">\n"""
     for i in items[0]:
          html_code += F'      <th>{i}</th>\n'
     html_code += "    </tr>\n  </thead>\n  <tbody>\n"
     for line in items[1:]:
          html_code += "    <tr>\n"
          for elt in line:
               _A    		=  F'{elt:.6f}' if isinstance(A__   ,		A__				) else str(A__				)
               html_code += F'      <td>{elt}</td>\n'
          html_code += "    </tr>\n"
     html_code += "  </tbody>\n</table><p>"
     return html_code







class lowercase_       :







   """simple docstring"""

   __lowerCAmelCase         =   5
   __lowerCAmelCase         =   0.2
   def __init__(      self      :			Optional[int], UpperCamelCase__      :			int, UpperCamelCase__      :			Optional[str] = None, UpperCamelCase__      :			bool = True, UpperCamelCase__      :			Optional["NotebookTrainingTracker"] = None, UpperCamelCase__      :			int = 3_00, )  ->			List[str]:
        _A    		=  total
        _A    		=  '' if prefix is None else prefix
        _A    		=  leave
        _A    		=  parent
        _A    		=  width
        _A    		=  None
        _A    		=  None
        _A    		=  None
   def 	__UpperCAmelCase   (      self      :			Union[str, Any], UpperCamelCase__      :			int, UpperCamelCase__      :			bool = False, UpperCamelCase__      :			str = None     )  ->			Any:
        _A    		=  value
        if comment is not None:
             _A    		=  comment
        if self.last_value is None:
             _A    		=  _A    		=  time.time()
             _A    		=  _A    		=  value
             _A    		=  _A    		=  None
             _A    		=  self.warmup
             _A    		=  1
             self.update_bar(__lowerCamelCase     )
        elif value <= self.last_value and not force_update:
             return
        elif force_update or self.first_calls > 0 or value >= min(self.last_value + self.wait_for, self.total     ):
             if self.first_calls > 0:
                  self.first_calls -= 1
             _A    		=  time.time()
             _A    		=  current_time - self.start_time
             # We could have value = self.start_value if the update is called twixe with the same start value.
             if value > self.start_value:
                  _A    		=  self.elapsed_time / (value - self.start_value)
             else:
                  _A    		=  None
             if value >= self.total:
                  _A    		=  self.total
                  _A    		=  None
                  if not self.leave:
                       self.close()
             elif self.average_time_per_item is not None:
                  _A    		=  self.average_time_per_item * (self.total - value)
             self.update_bar(__lowerCamelCase     )
             _A    		=  value
             _A    		=  current_time
             if self.average_time_per_item is None:
                  _A    		=  1
             else:
                  _A    		=  max(int(self.update_every / self.average_time_per_item     ), 1     )
   def 	__UpperCAmelCase   (      self      :			str, UpperCamelCase__      :			List[Any], UpperCamelCase__      :			List[Any]=None     )  ->			Any:
        _A    		=  ' ' * (len(str(self.total     )     ) - len(str(__lowerCamelCase     )     )) + str(__lowerCamelCase     )
        if self.elapsed_time is None:
             _A    		=  f'[{spaced_value}/{self.total} : < :'
        elif self.predicted_remaining is None:
             _A    		=  f'[{spaced_value}/{self.total} {format_time(self.elapsed_time     )}'
        else:
             _A    		=  (
                 f'[{spaced_value}/{self.total} {format_time(self.elapsed_time     )} <'
                 f' {format_time(self.predicted_remaining     )}'
             )
             self.label += f', {1/self.average_time_per_item:.2f} it/s'
        self.label += "]" if self.comment is None or len(self.comment     ) == 0 else f', {self.comment}]'
        self.display()
   def 	__UpperCAmelCase   (      self      :			Optional[Any]     )  ->			List[str]:
        _A    		=  html_progress_bar(self.value, self.total, self.prefix, self.label, self.width     )
        if self.parent is not None:
             # If this is a child bar, the parent will take care of the display.
             self.parent.display()
             return
        if self.output is None:
             _A    		=  disp.display(disp.HTML(self.html_code     ), display_id=__lowerCamelCase     )
        else:
             self.output.update(disp.HTML(self.html_code     )     )

   def 	__UpperCAmelCase   (      self      :			Union[str, Any]     )  ->			Optional[Any]:
        if self.parent is None and self.output is not None:
             self.output.update(disp.HTML(''     )     )







class lowercase_       (		__snake_case			):







   """simple docstring"""

   def __init__(      self      :			int, UpperCamelCase__      :			Optional[int], UpperCamelCase__      :			List[str]=None     )  ->			Optional[int]:
        super().__init__(__lowerCamelCase     )
        _A    		=  None if column_names is None else [column_names]
        _A    		=  None
   def 	__UpperCAmelCase   (      self      :			Any     )  ->			Any:
        _A    		=  html_progress_bar(self.value, self.total, self.prefix, self.label, self.width     )
        if self.inner_table is not None:
             self.html_code += text_to_html_table(self.inner_table     )
        if self.child_bar is not None:
             self.html_code += self.child_bar.html_code
        if self.output is None:
             _A    		=  disp.display(disp.HTML(self.html_code     ), display_id=__lowerCamelCase     )
        else:
             self.output.update(disp.HTML(self.html_code     )     )
   def 	__UpperCAmelCase   (      self      :			Optional[Any], UpperCamelCase__      :			Optional[Any]     )  ->			str:
        if self.inner_table is None:
             _A    		=  [list(values.keys()     ), list(values.values()     )]
        else:
             _A    		=  self.inner_table[0]
             if len(self.inner_table     ) == 1:
                  # We give a chance to update the column names at the first iteration
                  for key in values.keys():
                       if key not in columns:
                            columns.append(__lowerCamelCase     )
                  _A    		=  columns
             self.inner_table.append([values[c] for c in columns]     )
   def 	__UpperCAmelCase   (      self      :			Dict, UpperCamelCase__      :			Any, UpperCamelCase__      :			List[str]=None, UpperCamelCase__      :			str=3_00     )  ->			Optional[Any]:
        _A    		=  NotebookProgressBar(__lowerCamelCase, prefix=__lowerCamelCase, parent=self, width=__lowerCamelCase     )
        return self.child_bar

   def 	__UpperCAmelCase   (      self      :			Tuple     )  ->			Any:
        _A    		=  None
        self.display()







class lowercase_       (		__snake_case			):







   """simple docstring"""

   def __init__(      self      :			Dict     )  ->			List[Any]:
        _A    		=  None
        _A    		=  None
        _A    		=  False
   def 	__UpperCAmelCase   (      self      :			Optional[int], UpperCamelCase__      :			Optional[Any], UpperCamelCase__      :			str, UpperCamelCase__      :			str, **UpperCamelCase__      :			Tuple     )  ->			str:
        _A    		=  'Epoch' if args.evaluation_strategy == IntervalStrategy.EPOCH else 'Step'
        _A    		=  0
        _A    		=  0
        _A    		=  [self.first_column] + ['Training Loss']
        if args.evaluation_strategy != IntervalStrategy.NO:
             column_names.append('Validation Loss'     )
        _A    		=  NotebookTrainingTracker(state.max_steps, __lowerCamelCase     )
   def 	__UpperCAmelCase   (      self      :			Dict, UpperCamelCase__      :			Optional[Any], UpperCamelCase__      :			Dict, UpperCamelCase__      :			Tuple, **UpperCamelCase__      :			List[str]     )  ->			str:
        _A    		=  int(state.epoch     ) if int(state.epoch     ) == state.epoch else f'{state.epoch:.2f}'
        self.training_tracker.update(
            state.global_step + 1, comment=f'Epoch {epoch}/{state.num_train_epochs}', force_update=self._force_next_update, )
        _A    		=  False
   def 	__UpperCAmelCase   (      self      :			Dict, UpperCamelCase__      :			int, UpperCamelCase__      :			List[str], UpperCamelCase__      :			int, UpperCamelCase__      :			Any=None, **UpperCamelCase__      :			int     )  ->			str:
        if not has_length(__lowerCamelCase     ):
             return
        if self.prediction_bar is None:
             if self.training_tracker is not None:
                  _A    		=  self.training_tracker.add_child(len(__lowerCamelCase     )     )
             else:
                  _A    		=  NotebookProgressBar(len(__lowerCamelCase     )     )
             self.prediction_bar.update(1     )
        else:
             self.prediction_bar.update(self.prediction_bar.value + 1     )
   def 	__UpperCAmelCase   (      self      :			Union[str, Any], UpperCamelCase__      :			Optional[int], UpperCamelCase__      :			Union[str, Any], UpperCamelCase__      :			List[str], **UpperCamelCase__      :			List[Any]     )  ->			Any:
        if self.prediction_bar is not None:
             self.prediction_bar.close()
        _A    		=  None
   def 	__UpperCAmelCase   (      self      :			Optional[Any], UpperCamelCase__      :			str, UpperCamelCase__      :			int, UpperCamelCase__      :			Tuple, UpperCamelCase__      :			Dict=None, **UpperCamelCase__      :			Dict     )  ->			int:
        # Only for when there is no evaluation
        if args.evaluation_strategy == IntervalStrategy.NO and "loss" in logs:
             _A    		=  {'Training Loss': logs['loss']}
             # First column is necessarily Step sine we're not in epoch eval strategy
             _A    		=  state.global_step
             self.training_tracker.write_line(__lowerCamelCase     )
   def 	__UpperCAmelCase   (      self      :			str, UpperCamelCase__      :			str, UpperCamelCase__      :			List[Any], UpperCamelCase__      :			Any, UpperCamelCase__      :			Optional[int]=None, **UpperCamelCase__      :			int     )  ->			List[Any]:
        if self.training_tracker is not None:
             _A    		=  {'Training Loss': 'No log', 'Validation Loss': 'No log'}
             for log in reversed(state.log_history     ):
                  if "loss" in log:
                       _A    		=  log['loss']
                       break

             if self.first_column == "Epoch":
                  _A    		=  int(state.epoch     )
             else:
                  _A    		=  state.global_step
             _A    		=  'eval'
             for k in metrics:
                  if k.endswith('_loss'     ):
                       _A    		=  re.sub(r'\_loss$', '', __lowerCamelCase     )
             _A    		=  metrics.pop('total_flos', __lowerCamelCase     )
             _A    		=  metrics.pop('epoch', __lowerCamelCase     )
             _A    		=  metrics.pop(f'{metric_key_prefix}_runtime', __lowerCamelCase     )
             _A    		=  metrics.pop(f'{metric_key_prefix}_samples_per_second', __lowerCamelCase     )
             _A    		=  metrics.pop(f'{metric_key_prefix}_steps_per_second', __lowerCamelCase     )
             _A    		=  metrics.pop(f'{metric_key_prefix}_jit_compilation_time', __lowerCamelCase     )
             for k, v in metrics.items():
                  if k == f'{metric_key_prefix}_loss':
                       _A    		=  v
                  else:
                       _A    		=  k.split('_'     )
                       _A    		=  ' '.join([part.capitalize() for part in splits[1:]]     )
                       _A    		=  v
             self.training_tracker.write_line(__lowerCamelCase     )
             self.training_tracker.remove_child()
             _A    		=  None
             # Evaluation takes a long time so we should force the next update.
             _A    		=  True

   def 	__UpperCAmelCase   (      self      :			Union[str, Any], UpperCamelCase__      :			Dict, UpperCamelCase__      :			Dict, UpperCamelCase__      :			Dict, **UpperCamelCase__      :			Any     )  ->			Dict:
        self.training_tracker.update(
            state.global_step, comment=f'Epoch {int(state.epoch     )}/{state.num_train_epochs}', force_update=__lowerCamelCase     )
        _A    		=  None






from ...configuration_utils import PretrainedConfig
from ...utils import logging


__A			:					Optional[int]						=			logging.get_logger(__name__)

__A			:					Optional[Any]						=			{
    'EleutherAI/gpt-neox-20b': 'https://huggingface.co/EleutherAI/gpt-neox-20b/resolve/main/config.json',
    # See all GPTNeoX models at https://huggingface.co/models?filter=gpt_neox
}





class 			_SCREAMING_SNAKE_CASE (   __snake_case		):







   '''simple docstring'''




   lowerCamelCase__				=      "gpt_neox"





   def __init__(       self					:			Optional[int]     , __lowerCamelCase					:			List[str]=50432     , __lowerCamelCase					:			int=6144     , __lowerCamelCase					:			Optional[Any]=44     , __lowerCamelCase					:			Tuple=64     , __lowerCamelCase					:			Optional[int]=24576     , __lowerCamelCase					:			List[str]="gelu"     , __lowerCamelCase					:			Any=0.25     , __lowerCamelCase					:			List[Any]=10000     , __lowerCamelCase					:			List[Any]=0.0     , __lowerCamelCase					:			int=0.0     , __lowerCamelCase					:			str=0.1     , __lowerCamelCase					:			List[Any]=2048     , __lowerCamelCase					:			Tuple=0.02     , __lowerCamelCase					:			Tuple=1e-5     , __lowerCamelCase					:			Dict=True     , __lowerCamelCase					:			int=0     , __lowerCamelCase					:			Optional[Any]=2     , __lowerCamelCase					:			List[str]=False     , __lowerCamelCase					:			Optional[Any]=True     , __lowerCamelCase					:			Optional[int]=None     , **__lowerCamelCase					:			str     , ):
       super().__init__(bos_token_id=__lowerCamelCase     , eos_token_id=__lowerCamelCase     , **__lowerCamelCase			)
       SCREAMING_SNAKE_CASE										=   vocab_size
       SCREAMING_SNAKE_CASE										=   max_position_embeddings
       SCREAMING_SNAKE_CASE										=   hidden_size
       SCREAMING_SNAKE_CASE										=   num_hidden_layers
       SCREAMING_SNAKE_CASE										=   num_attention_heads
       SCREAMING_SNAKE_CASE										=   intermediate_size
       SCREAMING_SNAKE_CASE										=   hidden_act
       SCREAMING_SNAKE_CASE										=   rotary_pct
       SCREAMING_SNAKE_CASE										=   rotary_emb_base
       SCREAMING_SNAKE_CASE										=   attention_dropout
       SCREAMING_SNAKE_CASE										=   hidden_dropout
       SCREAMING_SNAKE_CASE										=   classifier_dropout
       SCREAMING_SNAKE_CASE										=   initializer_range
       SCREAMING_SNAKE_CASE										=   layer_norm_eps
       SCREAMING_SNAKE_CASE										=   use_cache
       SCREAMING_SNAKE_CASE										=   tie_word_embeddings
       SCREAMING_SNAKE_CASE										=   use_parallel_residual
       SCREAMING_SNAKE_CASE										=   rope_scaling
       self._rope_scaling_validation()

       if self.hidden_size % self.num_attention_heads != 0:
           raise ValueError(
               "The hidden size is not divisble by the number of attention heads! Make sure to update them!"			)





   def      _snake_case		(       self					:			Union[str, Any]			):
       if self.rope_scaling is None:
           return

       if not isinstance(self.rope_scaling     , __lowerCamelCase			) or len(self.rope_scaling			) != 2:
           raise ValueError(
               "`rope_scaling` must be a dictionary with with two fields, `name` and `factor`, "
               f"got {self.rope_scaling}"			)
       SCREAMING_SNAKE_CASE										=   self.rope_scaling.get("type"     , __lowerCamelCase			)
       SCREAMING_SNAKE_CASE										=   self.rope_scaling.get("factor"     , __lowerCamelCase			)
       if rope_scaling_type is None or rope_scaling_type not in ["linear", "dynamic"]:
           raise ValueError(
               f"`rope_scaling`'s name field must be one of ['linear', 'dynamic'], got {rope_scaling_type}"			)
       if rope_scaling_factor is None or not isinstance(__lowerCamelCase     , __lowerCamelCase			) or rope_scaling_factor <= 1.0:
           raise ValueError(f"`rope_scaling`'s factor field must be an float > 1, got {rope_scaling_factor}"			)