
'''simple docstring'''





import logging
import re

import pytorch_quantization
import pytorch_quantization.nn as quant_nn
import torch
from pytorch_quantization import calib
from pytorch_quantization.tensor_quant import QuantDescriptor


UpperCAmelCase    					=   logging.getLogger(__name__)

UpperCAmelCase    					=   50  # max width of layer names
UpperCAmelCase    					=   70  # max width of quantizer names

def      A (       A_							:					List[Any]       ):
						snake_case       :     Dict =    parser.add_argument_group('''quant_trainer arguments'''       )
						group.add_argument('''--wprec'''    ,							type=A_    ,							default=8    ,							help='''weight precision'''       )
						group.add_argument('''--aprec'''    ,							type=A_    ,							default=8    ,							help='''activation precision'''       )
						group.add_argument('''--quant-per-tensor'''    ,							action='''store_true'''    ,							help='''per tensor weight scaling'''       )
						group.add_argument('''--quant-disable'''    ,							action='''store_true'''    ,							help='''disable all quantizers'''       )
						group.add_argument('''--quant-disable-embeddings'''    ,							action='''store_true'''    ,							help='''disable all embeddings quantizers'''       )
						group.add_argument('''--quant-disable-keyword'''    ,							type=A_    ,							nargs='''+'''    ,							help='''disable quantizers by keyword'''       )
						group.add_argument('''--quant-disable-layer-module'''    ,							type=A_    ,							help='''disable quantizers by keyword under layer.'''       )
						group.add_argument('''--quant-enable-layer-module'''    ,							type=A_    ,							help='''enable quantizers by keyword under layer'''       )
						group.add_argument('''--calibrator'''    ,							default='''max'''    ,							help='''which quantization range calibrator to use'''       )
						group.add_argument('''--percentile'''    ,							default=A_    ,							type=A_    ,							help='''percentile for PercentileCalibrator'''       )
						group.add_argument('''--fuse-qkv'''    ,							action='''store_true'''    ,							help='''use the same scale factor for qkv'''       )
						group.add_argument('''--clip-gelu'''    ,							metavar='''N'''    ,							type=A_    ,							help='''clip gelu output maximum value to N'''       )
						group.add_argument(
						    '''--recalibrate-weights'''    ,							action='''store_true'''    ,							help=(
						        '''recalibrate weight amaxes by taking the max of the weights.'''
						        ''' amaxes will be computed with the current quantization granularity (axis).'''
						    )    ,							)

def      A (       A_							:					Union[str, Any]       ):

						if args.calibrator == "max":
												snake_case       :     Tuple =    '''max'''
						elif args.calibrator == "percentile":
												if args.percentile is None:
																		raise ValueError('''Specify --percentile when using percentile calibrator'''       )
												snake_case       :     List[Any] =    '''histogram'''
						elif args.calibrator == "mse":
												snake_case       :     int =    '''histogram'''
						else:
												raise ValueError(F"""Invalid calibrator {args.calibrator}"""       )

						snake_case       :     List[Any] =    QuantDescriptor(num_bits=args.aprec    ,							calib_method=A_       )
						snake_case       :     List[Any] =    QuantDescriptor(num_bits=args.wprec    ,							axis=(None if args.quant_per_tensor else (0,))       )
						quant_nn.QuantLinear.set_default_quant_desc_input(A_       )
						quant_nn.QuantLinear.set_default_quant_desc_weight(A_       )

def      A (       A_							:					Any    ,							A_							:					Union[str, Any]    ,							A_							:					Dict=False    ,							A_							:					Optional[Any]=False       ):
						logger.info('''Configuring Model for Quantization'''       )
						logger.info(F"""using quantization package {pytorch_quantization.__file__}"""       )

						if not calib:
												if args.quant_disable_embeddings:
																		set_quantizer_by_name(A_    ,							['''embeddings''']    ,							which='''weight'''    ,							_disabled=A_       )

												if args.quant_disable:
																		set_quantizer_by_name(A_    ,							['''''']    ,							_disabled=A_       )

												if args.quant_disable_keyword:
																		set_quantizer_by_name(A_    ,							args.quant_disable_keyword    ,							_disabled=A_       )

												if args.quant_disable_layer_module:
																		set_quantizer_by_name(A_    ,							[R'''layer.\d+.''' + args.quant_disable_layer_module]    ,							_disabled=A_       )

												if args.quant_enable_layer_module:
																		set_quantizer_by_name(A_    ,							[R'''layer.\d+.''' + args.quant_enable_layer_module]    ,							_disabled=A_       )

												if args.recalibrate_weights:
																		recalibrate_weights(A_       )

												if args.fuse_qkv:
																		fuse_qkv(A_    ,							A_       )

						if args.clip_gelu:
												clip_gelu(A_    ,							args.clip_gelu       )

						# if args.local_rank in [-1, 0] and not calib:
						print_quant_summary(A_       )

def      A (       A_							:					Union[str, Any]       ):
						logger.info('''Enabling Calibration'''       )
						for name, module in model.named_modules():
												if name.endswith('''_quantizer'''       ):
																		if module._calibrator is not None:
																								module.disable_quant()
																								module.enable_calib()
																		else:
																								module.disable()
																		logger.info(F"""{name:80}: {module}"""       )

def      A (       A_							:					List[Any]    ,							A_							:					Optional[int]       ):
						logger.info('''Loading calibrated amax'''       )
						for name, module in model.named_modules():
												if name.endswith('''_quantizer'''       ):
																		if module._calibrator is not None:
																								if isinstance(module._calibrator    ,							calib.MaxCalibrator       ):
																														module.load_calib_amax()
																								else:
																														module.load_calib_amax('''percentile'''    ,							percentile=args.percentile       )
																								module.enable_quant()
																								module.disable_calib()
																		else:
																								module.enable()
						model.cuda()
						print_quant_summary(A_       )

def      A (       A_							:					int    ,							A_							:					Optional[Any]       ):

						def fusea(A_							:					List[Any]    ,							A_							:					Dict    ,							A_							:					Optional[Any]       ):
												for mod in [qq, qk, qv]:
																		if not hasattr(A_    ,							'''_amax'''       ):
																								print('''          WARNING: NO AMAX BUFFER'''       )
																								return
												snake_case       :     List[str] =    qq._amax.detach().item()
												snake_case       :     Any =    qk._amax.detach().item()
												snake_case       :     Union[str, Any] =    qv._amax.detach().item()

												snake_case       :     Union[str, Any] =    max(A_    ,							A_    ,							A_       )
												qq._amax.fill_(A_       )
												qk._amax.fill_(A_       )
												qv._amax.fill_(A_       )
												logger.info(F"""          q={q:5.2f} k={k:5.2f} v={v:5.2f} -> {amax:5.2f}"""       )

						for name, mod in model.named_modules():
												if name.endswith('''.attention.self'''       ):
																		logger.info(F"""FUSE_QKV: {name:{name_width}}"""       )
																		fusea(mod.matmul_q_input_quantizer    ,							mod.matmul_k_input_quantizer    ,							mod.matmul_v_input_quantizer       )
																		if args.quant_per_tensor:
																								fusea(mod.query._weight_quantizer    ,							mod.key._weight_quantizer    ,							mod.value._weight_quantizer       )

def      A (       A_							:					Union[str, Any]    ,							A_							:					Tuple       ):

						for name, mod in model.named_modules():
												if name.endswith('''.output.dense'''       ) and not name.endswith('''attention.output.dense'''       ):
																		snake_case       :     Any =    mod._input_quantizer._amax.data.detach().item()
																		mod._input_quantizer._amax.data.detach().clamp_(max=A_       )
																		snake_case       :     List[str] =    mod._input_quantizer._amax.data.detach().item()
																		logger.info(F"""CLIP_GELU: {name:{name_width}} amax: {amax_init:5.2f} -> {amax:5.2f}"""       )

def      A (       A_							:					Optional[Any]       ):

						for name, mod in model.named_modules():
												if hasattr(A_    ,							'''_weight_quantizer'''       ) and mod._weight_quantizer.axis is not None:
																		snake_case       :     List[Any] =    mod.weight.shape[0]
																		snake_case       :     Dict =    mod._weight_quantizer._amax.detach()
																		snake_case       :     Union[str, Any] =    torch.ones(A_    ,							dtype=amax.dtype    ,							device=amax.device       ) * amax
																		print(F"""expanding {name} {amax} -> {mod._weight_quantizer._amax}"""       )

def      A (       A_							:					Optional[Any]       ):

						for name, mod in model.named_modules():
												if hasattr(A_    ,							'''_weight_quantizer'''       ):
																		if not hasattr(mod.weight_quantizer    ,							'''_amax'''       ):
																								print('''RECALIB: {name:{name_width}} WARNING: NO AMAX BUFFER'''       )
																								continue

																		# determine which axes to reduce across
																		# e.g. a 4D tensor quantized per axis 0 should reduce over (1,2,3)
																		snake_case       :     Optional[Any] =    set() if mod._weight_quantizer.axis is None else set(mod._weight_quantizer.axis       )
																		snake_case       :     Optional[Any] =    set(range(len(mod.weight.size()       )       )       ) - axis_set
																		snake_case       :     str =    pytorch_quantization.utils.reduce_amax(mod.weight    ,							axis=A_    ,							keepdims=A_       ).detach()
																		logger.info(F"""RECALIB: {name:{name_width}} {mod._weight_quantizer._amax.flatten()} -> {amax.flatten()}"""       )
																		snake_case       :     Any =    amax

def      A (       A_							:					List[str]    ,							A_							:					Any=25    ,							A_							:					List[str]=180    ,							A_							:					List[Any]=None       ):

						if ignore is None:
												snake_case       :     Tuple =    []
						elif not isinstance(A_    ,							A_       ):
												snake_case       :     str =    [ignore]

						snake_case       :     str =    0
						for name, mod in model.named_modules():
												if not hasattr(A_    ,							'''weight'''       ):
																		continue
												snake_case       :     Optional[Any] =    max(A_    ,							len(A_       )       )

						for name, mod in model.named_modules():
												snake_case       :     Optional[Any] =    getattr(A_    ,							'''_input_quantizer'''    ,							A_       )
												snake_case       :     List[Any] =    getattr(A_    ,							'''_weight_quantizer'''    ,							A_       )
												if not hasattr(A_    ,							'''weight'''       ):
																		continue
												if type(A_       ) in ignore:
																		continue
												if [True for s in ignore if type(A_       ) is str and s in name]:
																		continue
												snake_case       :     List[str] =    F"""Act:{input_q.extra_repr()}"""
												snake_case       :     List[Any] =    F"""Wgt:{weight_q.extra_repr()}"""
												snake_case       :     Dict =    F"""{name:{name_width}} {act_str} {wgt_str}"""
												if len(A_       ) <= line_width:
																		logger.info(A_       )
												else:
																		logger.info(F"""{name:{name_width}} {act_str}"""       )
																		logger.info(F"""{'  ':{name_width}} {wgt_str}"""       )

def      A (       A_							:					Union[str, Any]       ):
						snake_case       :     Dict =    0
						for name, mod in model.named_modules():
												if isinstance(A_    ,							pytorch_quantization.nn.TensorQuantizer       ):
																		print(F"""{name:80} {mod}"""       )
																		count += 1
						print(F"""{count} TensorQuantizers found in model"""       )

def      A (       A_							:					List[Any]    ,							A_							:					Optional[int]    ,							A_							:					int    ,							A_							:					str    ,							A_							:					int       ):
						snake_case       :     Tuple =    getattr(A_    ,							A_    ,							A_       )
						if quantizer_mod is not None:
												assert hasattr(A_    ,							A_       )
												setattr(A_    ,							A_    ,							A_       )
						else:
												logger.warning(F"""{name} has no {quantizer}"""       )

def      A (       A_							:					List[Any]    ,							A_							:					List[Any]    ,							A_							:					Optional[int]="both"    ,							**A_							:					Dict       ):
						snake_case       :     int =    F"""Warning: changing {which} quantizers of {name:{qname_width}}"""
						for k, v in kwargs.items():
												s += F""" {k}={v}"""
												if which in ["input", "both"]:
																		set_quantizer(A_    ,							A_    ,							'''_input_quantizer'''    ,							A_    ,							A_       )
												if which in ["weight", "both"]:
																		set_quantizer(A_    ,							A_    ,							'''_weight_quantizer'''    ,							A_    ,							A_       )
						logger.info(A_       )





def      A (       A_							:					List[str]    ,							A_							:					int    ,							**A_							:					Optional[int]       ):

						for name, mod in model.named_modules():
												if hasattr(A_    ,							'''_input_quantizer'''       ) or hasattr(A_    ,							'''_weight_quantizer'''       ):
																		for n in names:
																								if re.search(A_    ,							A_       ):
																														set_quantizers(A_    ,							A_    ,							**A_       )
												elif name.endswith('''_quantizer'''       ):
																		for n in names:
																								if re.search(A_    ,							A_       ):
																														snake_case       :     List[Any] =    F"""Warning: changing {name:{name_width}}"""
																														for k, v in kwargs.items():
																																				s += F""" {k}={v}"""
																																				setattr(A_    ,							A_    ,							A_       )
																														logger.info(A_       )



'''simple docstring'''





import re
import time
from typing import Optional

import IPython.display as disp

from ..trainer_callback import TrainerCallback
from ..trainer_utils import IntervalStrategy, has_length

def      A (       A_							:					str       ):
						snake_case       :     List[str] =    int(A_       )
						snake_case,					snake_case,					snake_case       :     Any =    t // 3600, (t // 60) % 60, t % 60
						return F"""{h}:{m:02d}:{s:02d}""" if h != 0 else F"""{m:02d}:{s:02d}"""

def      A (       A_							:					Optional[int]    ,							A_							:					int    ,							A_							:					List[str]    ,							A_							:					Optional[int]    ,							A_							:					Optional[int]=300       ):
						# docstyle-ignore
						return F"""
    <div>
      {prefix}
      <progress value='{value}' max='{total}' style='width:{width}px; height:20px; vertical-align: middle;'></progress>
      {label}
    </div>
    """

def      A (       A_							:					Dict       ):
						snake_case       :     Tuple =    '''<table border="1" class="dataframe">\n'''
						html_code += """  <thead>\n <tr style="text-align: left;">\n"""
						for i in items[0]:
												html_code += F"""      <th>{i}</th>\n"""
						html_code += "    </tr>\n  </thead>\n  <tbody>\n"
						for line in items[1:]:
												html_code += "    <tr>\n"
												for elt in line:
																		snake_case       :     List[str] =    F"""{elt:.6f}""" if isinstance(A_    ,							A_       ) else str(A_       )
																		html_code += F"""      <td>{elt}</td>\n"""
												html_code += "    </tr>\n"
						html_code += "  </tbody>\n</table><p>"
						return html_code

class 						a   :
				_snake_case		    = 5
				_snake_case		    = 0.2




				def __init__(							self	:				Union[str, Any],    SCREAMING_SNAKE_CASE_	:				int,    SCREAMING_SNAKE_CASE_	:				Optional[str] = None,    SCREAMING_SNAKE_CASE_	:				bool = True,    SCREAMING_SNAKE_CASE_	:				Optional["NotebookTrainingTracker"] = None,    SCREAMING_SNAKE_CASE_	:				int = 3_00,    ):
										snake_case       :     List[Any] =    total
										snake_case       :     Union[str, Any] =    '''''' if prefix is None else prefix
										snake_case       :     List[Any] =    leave
										snake_case       :     int =    parent
										snake_case       :     List[str] =    width
										snake_case       :     Optional[int] =    None
										snake_case       :     Optional[Any] =    None
										snake_case       :     Tuple =    None




				def 	__snake_case							(							self	:				Any,    SCREAMING_SNAKE_CASE_	:				int,    SCREAMING_SNAKE_CASE_	:				bool = False,    SCREAMING_SNAKE_CASE_	:				str = None      ):
										snake_case       :     List[Any] =    value
										if comment is not None:
																snake_case       :     Tuple =    comment
										if self.last_value is None:
																snake_case       :     str =    time.time()
																snake_case       :     List[str] =    value
																snake_case       :     str =    None
																snake_case       :     Dict =    self.warmup
																snake_case       :     Tuple =    1
																self.update_bar(SCREAMING_SNAKE_CASE_      )
										elif value <= self.last_value and not force_update:
																return
										elif force_update or self.first_calls > 0 or value >= min(self.last_value + self.wait_for,    self.total      ):
																if self.first_calls > 0:
																						self.first_calls -= 1
																snake_case       :     Tuple =    time.time()
																snake_case       :     str =    current_time - self.start_time
																# We could have value = self.start_value if the update is called twixe with the same start value.
																if value > self.start_value:
																						snake_case       :     Any =    self.elapsed_time / (value - self.start_value)
																else:
																						snake_case       :     List[str] =    None
																if value >= self.total:
																						snake_case       :     List[Any] =    self.total
																						snake_case       :     Tuple =    None
																						if not self.leave:
																												self.close()
																elif self.average_time_per_item is not None:
																						snake_case       :     Optional[int] =    self.average_time_per_item * (self.total - value)
																self.update_bar(SCREAMING_SNAKE_CASE_      )
																snake_case       :     List[str] =    value
																snake_case       :     List[Any] =    current_time
																if self.average_time_per_item is None:
																						snake_case       :     List[str] =    1
																else:
																						snake_case       :     Optional[int] =    max(int(self.update_every / self.average_time_per_item      ),    1      )




				def 	__snake_case							(							self	:				Any,    SCREAMING_SNAKE_CASE_	:				List[str],    SCREAMING_SNAKE_CASE_	:				Optional[Any]=None      ):
										snake_case       :     Optional[Any] =    ''' ''' * (len(str(self.total      )      ) - len(str(SCREAMING_SNAKE_CASE_      )      )) + str(SCREAMING_SNAKE_CASE_      )
										if self.elapsed_time is None:
																snake_case       :     int =    F"""[{spaced_value}/{self.total} : < :"""
										elif self.predicted_remaining is None:
																snake_case       :     List[str] =    F"""[{spaced_value}/{self.total} {format_time(self.elapsed_time      )}"""
										else:
																snake_case       :     Optional[int] =    (
																    F"""[{spaced_value}/{self.total} {format_time(self.elapsed_time      )} <"""
																    F""" {format_time(self.predicted_remaining      )}"""
																)
																self.label += F""", {1/self.average_time_per_item:.2f} it/s"""
										self.label += "]" if self.comment is None or len(self.comment      ) == 0 else F""", {self.comment}]"""
										self.display()




				def 	__snake_case							(							self	:				Optional[int]      ):
										snake_case       :     str =    html_progress_bar(self.value,    self.total,    self.prefix,    self.label,    self.width      )
										if self.parent is not None:
																# If this is a child bar, the parent will take care of the display.
																self.parent.display()
																return
										if self.output is None:
																snake_case       :     Any =    disp.display(disp.HTML(self.html_code      ),    display_id=SCREAMING_SNAKE_CASE_      )
										else:
																self.output.update(disp.HTML(self.html_code      )      )




				def 	__snake_case							(							self	:				Optional[int]      ):
										if self.parent is None and self.output is not None:
																self.output.update(disp.HTML(''''''      )      )




class 						a   (				__magic_name__    ):




				def __init__(							self	:				List[str],    SCREAMING_SNAKE_CASE_	:				List[str],    SCREAMING_SNAKE_CASE_	:				Optional[int]=None      ):
										super().__init__(SCREAMING_SNAKE_CASE_      )
										snake_case       :     Tuple =    None if column_names is None else [column_names]
										snake_case       :     str =    None




				def 	__snake_case							(							self	:				Dict      ):
										snake_case       :     Tuple =    html_progress_bar(self.value,    self.total,    self.prefix,    self.label,    self.width      )
										if self.inner_table is not None:
																self.html_code += text_to_html_table(self.inner_table      )
										if self.child_bar is not None:
																self.html_code += self.child_bar.html_code
										if self.output is None:
																snake_case       :     List[Any] =    disp.display(disp.HTML(self.html_code      ),    display_id=SCREAMING_SNAKE_CASE_      )
										else:
																self.output.update(disp.HTML(self.html_code      )      )




				def 	__snake_case							(							self	:				List[str],    SCREAMING_SNAKE_CASE_	:				Any      ):
										if self.inner_table is None:
																snake_case       :     Optional[Any] =    [list(values.keys()      ), list(values.values()      )]
										else:
																snake_case       :     Tuple =    self.inner_table[0]
																if len(self.inner_table      ) == 1:
																						# We give a chance to update the column names at the first iteration
																						for key in values.keys():
																												if key not in columns:
																																		columns.append(SCREAMING_SNAKE_CASE_      )
																						snake_case       :     int =    columns
																self.inner_table.append([values[c] for c in columns]      )




				def 	__snake_case							(							self	:				Tuple,    SCREAMING_SNAKE_CASE_	:				Optional[Any],    SCREAMING_SNAKE_CASE_	:				int=None,    SCREAMING_SNAKE_CASE_	:				str=3_00      ):
										snake_case       :     int =    NotebookProgressBar(SCREAMING_SNAKE_CASE_,    prefix=SCREAMING_SNAKE_CASE_,    parent=self,    width=SCREAMING_SNAKE_CASE_      )
										return self.child_bar




				def 	__snake_case							(							self	:				Union[str, Any]      ):
										snake_case       :     int =    None
										self.display()




class 						a   (				__magic_name__    ):




				def __init__(							self	:				Optional[Any]      ):
										snake_case       :     Dict =    None
										snake_case       :     Union[str, Any] =    None
										snake_case       :     Dict =    False




				def 	__snake_case							(							self	:				List[str],    SCREAMING_SNAKE_CASE_	:				Any,    SCREAMING_SNAKE_CASE_	:				int,    SCREAMING_SNAKE_CASE_	:				str,    **SCREAMING_SNAKE_CASE_	:				List[str]      ):
										snake_case       :     str =    '''Epoch''' if args.evaluation_strategy == IntervalStrategy.EPOCH else '''Step'''
										snake_case       :     List[str] =    0
										snake_case       :     Dict =    0
										snake_case       :     List[Any] =    [self.first_column] + ['''Training Loss''']
										if args.evaluation_strategy != IntervalStrategy.NO:
																column_names.append('''Validation Loss'''      )
										snake_case       :     Tuple =    NotebookTrainingTracker(state.max_steps,    SCREAMING_SNAKE_CASE_      )




				def 	__snake_case							(							self	:				Tuple,    SCREAMING_SNAKE_CASE_	:				List[Any],    SCREAMING_SNAKE_CASE_	:				Dict,    SCREAMING_SNAKE_CASE_	:				str,    **SCREAMING_SNAKE_CASE_	:				Union[str, Any]      ):
										snake_case       :     Tuple =    int(state.epoch      ) if int(state.epoch      ) == state.epoch else F"""{state.epoch:.2f}"""
										self.training_tracker.update(
										    state.global_step + 1,    comment=F"""Epoch {epoch}/{state.num_train_epochs}""",    force_update=self._force_next_update,    )
										snake_case       :     str =    False




				def 	__snake_case							(							self	:				Tuple,    SCREAMING_SNAKE_CASE_	:				int,    SCREAMING_SNAKE_CASE_	:				Union[str, Any],    SCREAMING_SNAKE_CASE_	:				int,    SCREAMING_SNAKE_CASE_	:				Tuple=None,    **SCREAMING_SNAKE_CASE_	:				Tuple      ):
										if not has_length(SCREAMING_SNAKE_CASE_      ):
																return
										if self.prediction_bar is None:
																if self.training_tracker is not None:
																						snake_case       :     Dict =    self.training_tracker.add_child(len(SCREAMING_SNAKE_CASE_      )      )
																else:
																						snake_case       :     Optional[Any] =    NotebookProgressBar(len(SCREAMING_SNAKE_CASE_      )      )
																self.prediction_bar.update(1      )
										else:
																self.prediction_bar.update(self.prediction_bar.value + 1      )




				def 	__snake_case							(							self	:				Optional[int],    SCREAMING_SNAKE_CASE_	:				Any,    SCREAMING_SNAKE_CASE_	:				Dict,    SCREAMING_SNAKE_CASE_	:				Union[str, Any],    **SCREAMING_SNAKE_CASE_	:				List[Any]      ):
										if self.prediction_bar is not None:
																self.prediction_bar.close()
										snake_case       :     Optional[int] =    None




				def 	__snake_case							(							self	:				Optional[Any],    SCREAMING_SNAKE_CASE_	:				List[str],    SCREAMING_SNAKE_CASE_	:				Any,    SCREAMING_SNAKE_CASE_	:				Optional[int],    SCREAMING_SNAKE_CASE_	:				Optional[int]=None,    **SCREAMING_SNAKE_CASE_	:				Union[str, Any]      ):
										# Only for when there is no evaluation
										if args.evaluation_strategy == IntervalStrategy.NO and "loss" in logs:
																snake_case       :     List[str] =    {'''Training Loss''': logs['''loss''']}
																# First column is necessarily Step sine we're not in epoch eval strategy
																snake_case       :     List[Any] =    state.global_step
																self.training_tracker.write_line(SCREAMING_SNAKE_CASE_      )




				def 	__snake_case							(							self	:				Any,    SCREAMING_SNAKE_CASE_	:				Optional[int],    SCREAMING_SNAKE_CASE_	:				List[str],    SCREAMING_SNAKE_CASE_	:				List[str],    SCREAMING_SNAKE_CASE_	:				Optional[Any]=None,    **SCREAMING_SNAKE_CASE_	:				Any      ):
										if self.training_tracker is not None:
																snake_case       :     List[Any] =    {'''Training Loss''': '''No log''', '''Validation Loss''': '''No log'''}
																for log in reversed(state.log_history      ):
																						if "loss" in log:
																												snake_case       :     Union[str, Any] =    log['''loss''']
																												break

																if self.first_column == "Epoch":
																						snake_case       :     Optional[int] =    int(state.epoch      )
																else:
																						snake_case       :     Optional[int] =    state.global_step
																snake_case       :     Optional[int] =    '''eval'''
																for k in metrics:
																						if k.endswith('''_loss'''      ):
																												snake_case       :     str =    re.sub(R'''\_loss$''',    '''''',    SCREAMING_SNAKE_CASE_      )
																snake_case       :     str =    metrics.pop('''total_flos''',    SCREAMING_SNAKE_CASE_      )
																snake_case       :     int =    metrics.pop('''epoch''',    SCREAMING_SNAKE_CASE_      )
																snake_case       :     List[str] =    metrics.pop(F"""{metric_key_prefix}_runtime""",    SCREAMING_SNAKE_CASE_      )
																snake_case       :     Union[str, Any] =    metrics.pop(F"""{metric_key_prefix}_samples_per_second""",    SCREAMING_SNAKE_CASE_      )
																snake_case       :     Any =    metrics.pop(F"""{metric_key_prefix}_steps_per_second""",    SCREAMING_SNAKE_CASE_      )
																snake_case       :     Dict =    metrics.pop(F"""{metric_key_prefix}_jit_compilation_time""",    SCREAMING_SNAKE_CASE_      )
																for k, v in metrics.items():
																						if k == F"""{metric_key_prefix}_loss""":
																												snake_case       :     Optional[int] =    v
																						else:
																												snake_case       :     int =    k.split('''_'''      )
																												snake_case       :     Any =    ''' '''.join([part.capitalize() for part in splits[1:]]      )
																												snake_case       :     Tuple =    v
																self.training_tracker.write_line(SCREAMING_SNAKE_CASE_      )
																self.training_tracker.remove_child()
																snake_case       :     Any =    None
																# Evaluation takes a long time so we should force the next update.
																snake_case       :     Optional[int] =    True




				def 	__snake_case							(							self	:				List[str],    SCREAMING_SNAKE_CASE_	:				List[Any],    SCREAMING_SNAKE_CASE_	:				List[str],    SCREAMING_SNAKE_CASE_	:				Dict,    **SCREAMING_SNAKE_CASE_	:				str      ):
										self.training_tracker.update(
										    state.global_step,    comment=F"""Epoch {int(state.epoch      )}/{state.num_train_epochs}""",    force_update=SCREAMING_SNAKE_CASE_      )
										snake_case       :     Tuple =    None

