





"""simple docstring"""


import logging
import re

import pytorch_quantization
import pytorch_quantization.nn as quant_nn
import torch
from pytorch_quantization import calib
from pytorch_quantization.tensor_quant import QuantDescriptor


A_	: Union[str, Any]      				=logging.getLogger(__name__)

A_	: Optional[Any]      				=5_0  # max width of layer names
A_	: Optional[int]      				=7_0  # max width of quantizer names





def SCREAMING_SNAKE_CASE_  (			snake_case  :   int						)->						Tuple:
       _lowerCamelCase             =   parser.add_argument_group('quant_trainer arguments'						)
       group.add_argument('--wprec'		,    type=snake_case		,    default=8		,    help='weight precision'						)
       group.add_argument('--aprec'		,    type=snake_case		,    default=8		,    help='activation precision'						)
       group.add_argument('--quant-per-tensor'		,    action='store_true'		,    help='per tensor weight scaling'						)
       group.add_argument('--quant-disable'		,    action='store_true'		,    help='disable all quantizers'						)
       group.add_argument('--quant-disable-embeddings'		,    action='store_true'		,    help='disable all embeddings quantizers'						)
       group.add_argument('--quant-disable-keyword'		,    type=snake_case		,    nargs='+'		,    help='disable quantizers by keyword'						)
       group.add_argument('--quant-disable-layer-module'		,    type=snake_case		,    help='disable quantizers by keyword under layer.'						)
       group.add_argument('--quant-enable-layer-module'		,    type=snake_case		,    help='enable quantizers by keyword under layer'						)
       group.add_argument('--calibrator'		,    default='max'		,    help='which quantization range calibrator to use'						)
       group.add_argument('--percentile'		,    default=snake_case		,    type=snake_case		,    help='percentile for PercentileCalibrator'						)
       group.add_argument('--fuse-qkv'		,    action='store_true'		,    help='use the same scale factor for qkv'						)
       group.add_argument('--clip-gelu'		,    metavar='N'		,    type=snake_case		,    help='clip gelu output maximum value to N'						)
       group.add_argument(
           '--recalibrate-weights'		,    action='store_true'		,    help=(
               'recalibrate weight amaxes by taking the max of the weights.'
               ' amaxes will be computed with the current quantization granularity (axis).'
           )		,    )





def SCREAMING_SNAKE_CASE_  (			snake_case  :   Dict						)->						Union[str, Any]:

       if args.calibrator == "max":
              _lowerCamelCase             =   """max"""
       elif args.calibrator == "percentile":
              if args.percentile is None:
                     raise ValueError('Specify --percentile when using percentile calibrator'						)
              _lowerCamelCase             =   """histogram"""
       elif args.calibrator == "mse":
              _lowerCamelCase             =   """histogram"""
       else:
              raise ValueError(f'Invalid calibrator {args.calibrator}'						)

       _lowerCamelCase             =   QuantDescriptor(num_bits=args.aprec		,    calib_method=snake_case						)
       _lowerCamelCase             =   QuantDescriptor(num_bits=args.wprec		,    axis=(None if args.quant_per_tensor else (0,))						)
       quant_nn.QuantLinear.set_default_quant_desc_input(snake_case						)
       quant_nn.QuantLinear.set_default_quant_desc_weight(snake_case						)





def SCREAMING_SNAKE_CASE_  (			snake_case  :   Optional[int]		,    snake_case  :   List[str]		,    snake_case  :   Tuple=False		,    snake_case  :   Dict=False						)->						Dict:
       logger.info('Configuring Model for Quantization'						)
       logger.info(f'using quantization package {pytorch_quantization.__file__}'						)

       if not calib:
              if args.quant_disable_embeddings:
                     set_quantizer_by_name(snake_case		,    ['embeddings']		,    which='weight'		,    _disabled=snake_case						)

              if args.quant_disable:
                     set_quantizer_by_name(snake_case		,    ['']		,    _disabled=snake_case						)

              if args.quant_disable_keyword:
                     set_quantizer_by_name(snake_case		,    args.quant_disable_keyword		,    _disabled=snake_case						)

              if args.quant_disable_layer_module:
                     set_quantizer_by_name(snake_case		,    [r'layer.\d+.' + args.quant_disable_layer_module]		,    _disabled=snake_case						)

              if args.quant_enable_layer_module:
                     set_quantizer_by_name(snake_case		,    [r'layer.\d+.' + args.quant_enable_layer_module]		,    _disabled=snake_case						)

              if args.recalibrate_weights:
                     recalibrate_weights(snake_case						)

              if args.fuse_qkv:
                     fuse_qkv(snake_case		,    snake_case						)

       if args.clip_gelu:
              clip_gelu(snake_case		,    args.clip_gelu						)

       # if args.local_rank in [-1, 0] and not calib:
       print_quant_summary(snake_case						)





def SCREAMING_SNAKE_CASE_  (			snake_case  :   Union[str, Any]						)->						Tuple:
       logger.info('Enabling Calibration'						)
       for name, module in model.named_modules():
              if name.endswith('_quantizer'						):
                     if module._calibrator is not None:
                            module.disable_quant()
                            module.enable_calib()
                     else:
                            module.disable()
                     logger.info(f'{name:80}: {module}'						)





def SCREAMING_SNAKE_CASE_  (			snake_case  :   List[Any]		,    snake_case  :   Optional[Any]						)->						Dict:
       logger.info('Loading calibrated amax'						)
       for name, module in model.named_modules():
              if name.endswith('_quantizer'						):
                     if module._calibrator is not None:
                            if isinstance(module._calibrator		,    calib.MaxCalibrator						):
                                   module.load_calib_amax()
                            else:
                                   module.load_calib_amax('percentile'		,    percentile=args.percentile						)
                            module.enable_quant()
                            module.disable_calib()
                     else:
                            module.enable()
       model.cuda()
       print_quant_summary(snake_case						)





def SCREAMING_SNAKE_CASE_  (			snake_case  :   Tuple		,    snake_case  :   List[str]						)->						Tuple:

       def fusea(snake_case  :   Union[str, Any]		,    snake_case  :   Optional[Any]		,    snake_case  :   List[str]						):
              for mod in [qq, qk, qv]:
                     if not hasattr(snake_case		,    '_amax'						):
                            print('          WARNING: NO AMAX BUFFER'						)
                            return
              _lowerCamelCase             =   qq._amax.detach().item()
              _lowerCamelCase             =   qk._amax.detach().item()
              _lowerCamelCase             =   qv._amax.detach().item()

              _lowerCamelCase             =   max(snake_case		,    snake_case		,    snake_case						)
              qq._amax.fill_(snake_case						)
              qk._amax.fill_(snake_case						)
              qv._amax.fill_(snake_case						)
              logger.info(f'          q={q:5.2f} k={k:5.2f} v={v:5.2f} -> {amax:5.2f}'						)

       for name, mod in model.named_modules():
              if name.endswith('.attention.self'						):
                     logger.info(f'FUSE_QKV: {name:{name_width}}'						)
                     fusea(mod.matmul_q_input_quantizer		,    mod.matmul_k_input_quantizer		,    mod.matmul_v_input_quantizer						)
                     if args.quant_per_tensor:
                            fusea(mod.query._weight_quantizer		,    mod.key._weight_quantizer		,    mod.value._weight_quantizer						)





def SCREAMING_SNAKE_CASE_  (			snake_case  :   List[Any]		,    snake_case  :   Optional[int]						)->						Optional[Any]:

       for name, mod in model.named_modules():
              if name.endswith('.output.dense'						) and not name.endswith('attention.output.dense'						):
                     _lowerCamelCase             =   mod._input_quantizer._amax.data.detach().item()
                     mod._input_quantizer._amax.data.detach().clamp_(max=snake_case						)
                     _lowerCamelCase             =   mod._input_quantizer._amax.data.detach().item()
                     logger.info(f'CLIP_GELU: {name:{name_width}} amax: {amax_init:5.2f} -> {amax:5.2f}'						)





def SCREAMING_SNAKE_CASE_  (			snake_case  :   Dict						)->						List[Any]:

       for name, mod in model.named_modules():
              if hasattr(snake_case		,    '_weight_quantizer'						) and mod._weight_quantizer.axis is not None:
                     _lowerCamelCase             =   mod.weight.shape[0]
                     _lowerCamelCase             =   mod._weight_quantizer._amax.detach()
                     _lowerCamelCase             =   torch.ones(snake_case		,    dtype=amax.dtype		,    device=amax.device						) * amax
                     print(f'expanding {name} {amax} -> {mod._weight_quantizer._amax}'						)





def SCREAMING_SNAKE_CASE_  (			snake_case  :   str						)->						Any:

       for name, mod in model.named_modules():
              if hasattr(snake_case		,    '_weight_quantizer'						):
                     if not hasattr(mod.weight_quantizer		,    '_amax'						):
                            print('RECALIB: {name:{name_width}} WARNING: NO AMAX BUFFER'						)
                            continue

                     # determine which axes to reduce across
                     # e.g. a 4D tensor quantized per axis 0 should reduce over (1,2,3)
                     _lowerCamelCase             =   set() if mod._weight_quantizer.axis is None else set(mod._weight_quantizer.axis						)
                     _lowerCamelCase             =   set(range(len(mod.weight.size()						)						)						) - axis_set
                     _lowerCamelCase             =   pytorch_quantization.utils.reduce_amax(mod.weight		,    axis=snake_case		,    keepdims=snake_case						).detach()
                     logger.info(f'RECALIB: {name:{name_width}} {mod._weight_quantizer._amax.flatten()} -> {amax.flatten()}'						)
                     _lowerCamelCase             =   amax





def SCREAMING_SNAKE_CASE_  (			snake_case  :   Union[str, Any]		,    snake_case  :   str=25		,    snake_case  :   int=180		,    snake_case  :   Any=None						)->						Optional[Any]:

       if ignore is None:
              _lowerCamelCase             =   []
       elif not isinstance(snake_case		,    snake_case						):
              _lowerCamelCase             =   [ignore]

       _lowerCamelCase             =   0
       for name, mod in model.named_modules():
              if not hasattr(snake_case		,    'weight'						):
                     continue
              _lowerCamelCase             =   max(snake_case		,    len(snake_case						)						)

       for name, mod in model.named_modules():
              _lowerCamelCase             =   getattr(snake_case		,    '_input_quantizer'		,    snake_case						)
              _lowerCamelCase             =   getattr(snake_case		,    '_weight_quantizer'		,    snake_case						)
              if not hasattr(snake_case		,    'weight'						):
                     continue
              if type(snake_case						) in ignore:
                     continue
              if [True for s in ignore if type(snake_case						) is str and s in name]:
                     continue
              _lowerCamelCase             =   f'Act:{input_q.extra_repr()}'
              _lowerCamelCase             =   f'Wgt:{weight_q.extra_repr()}'
              _lowerCamelCase             =   f'{name:{name_width}} {act_str} {wgt_str}'
              if len(snake_case						) <= line_width:
                     logger.info(snake_case						)
              else:
                     logger.info(f'{name:{name_width}} {act_str}'						)
                     logger.info(f'{"  ":{name_width}} {wgt_str}'						)





def SCREAMING_SNAKE_CASE_  (			snake_case  :   List[Any]						)->						Optional[int]:
       _lowerCamelCase             =   0
       for name, mod in model.named_modules():
              if isinstance(snake_case		,    pytorch_quantization.nn.TensorQuantizer						):
                     print(f'{name:80} {mod}'						)
                     count += 1
       print(f'{count} TensorQuantizers found in model'						)





def SCREAMING_SNAKE_CASE_  (			snake_case  :   str		,    snake_case  :   int		,    snake_case  :   Optional[int]		,    snake_case  :   List[Any]		,    snake_case  :   str						)->						Any:
       _lowerCamelCase             =   getattr(snake_case		,    snake_case		,    snake_case						)
       if quantizer_mod is not None:
              assert hasattr(snake_case		,    snake_case						)
              setattr(snake_case		,    snake_case		,    snake_case						)
       else:
              logger.warning(f'{name} has no {quantizer}'						)





def SCREAMING_SNAKE_CASE_  (			snake_case  :   str		,    snake_case  :   Optional[Any]		,    snake_case  :   Dict="both"		,    **snake_case  :   List[str]						)->						Tuple:
       _lowerCamelCase             =   f'Warning: changing {which} quantizers of {name:{qname_width}}'
       for k, v in kwargs.items():
              s += f' {k}={v}'
              if which in ["input", "both"]:
                     set_quantizer(snake_case		,    snake_case		,    '_input_quantizer'		,    snake_case		,    snake_case						)
              if which in ["weight", "both"]:
                     set_quantizer(snake_case		,    snake_case		,    '_weight_quantizer'		,    snake_case		,    snake_case						)
       logger.info(snake_case						)





def SCREAMING_SNAKE_CASE_  (			snake_case  :   int		,    snake_case  :   List[Any]		,    **snake_case  :   int						)->						Optional[Any]:

       for name, mod in model.named_modules():
              if hasattr(snake_case		,    '_input_quantizer'						) or hasattr(snake_case		,    '_weight_quantizer'						):
                     for n in names:
                            if re.search(snake_case		,    snake_case						):
                                   set_quantizers(snake_case		,    snake_case		,    **snake_case						)
              elif name.endswith('_quantizer'						):
                     for n in names:
                            if re.search(snake_case		,    snake_case						):
                                   _lowerCamelCase             =   f'Warning: changing {name:{name_width}}'
                                   for k, v in kwargs.items():
                                          s += f' {k}={v}'
                                          setattr(snake_case		,    snake_case		,    snake_case						)
                                   logger.info(snake_case						)












from typing import Dict, List, Optional, Union

import numpy as np

from .feature_extraction_utils import BatchFeature, FeatureExtractionMixin
from .utils import PaddingStrategy, TensorType, is_tf_tensor, is_torch_tensor, logging, to_numpy


UpperCAmelCase_		:					Dict          =  logging.get_logger(__name__)




class 		__A   (      UpperCamelCase__      ):

						def __init__(		self :List[str]     ,			__snake_case :int     ,			__snake_case :int     ,			__snake_case :float     ,			**__snake_case :Optional[Any]			):
										'''simple docstring'''

										__magic_name__						:						List[Any]										=feature_size
										__magic_name__						:						Union[str, Any]										=sampling_rate
										__magic_name__						:						List[Any]										=padding_value

										__magic_name__						:						List[str]										=kwargs.pop("""padding_side"""     ,			"""right"""			)
										__magic_name__						:						Tuple										=kwargs.pop("""return_attention_mask"""     ,			__snake_case			)

										super().__init__(**__snake_case			)

						def      A__    (		self :Any     ,			__snake_case :Union[
						        BatchFeature,
						        List[BatchFeature],
						        Dict[str, BatchFeature],
						        Dict[str, List[BatchFeature]],
						        List[Dict[str, BatchFeature]],
						    ]     ,			__snake_case :Union[bool, str, PaddingStrategy] = True     ,			__snake_case :Optional[int] = None     ,			__snake_case :bool = False     ,			__snake_case :Optional[int] = None     ,			__snake_case :Optional[bool] = None     ,			__snake_case :Optional[Union[str, TensorType]] = None     ,			):
										'''simple docstring'''

										if isinstance(__snake_case     ,			(list, tuple)			) and isinstance(processed_features[0]     ,			(dict, BatchFeature)			):
														__magic_name__						:						Union[str, Any]										={
														    key: [example[key] for example in processed_features] for key in processed_features[0].keys()
														}

										# The model's main input name, usually `input_values`, has be passed for padding
										if self.model_input_names[0] not in processed_features:
														raise ValueError(
														    """You should supply an instance of `transformers.BatchFeature` or list of `transformers.BatchFeature`"""
														    f" to this method that includes {self.model_input_names[0]}, but you provided"
														    f" {list(processed_features.keys()			)}"			)

										__magic_name__						:						int										=processed_features[self.model_input_names[0]]
										__magic_name__						:						Union[str, Any]										=(
										    return_attention_mask if return_attention_mask is not None else self.return_attention_mask
										)

										if len(__snake_case			) == 0:
														if return_attention_mask:
																		__magic_name__						:						List[str]										=[]
														return processed_features

										# If we have PyTorch/TF tensors or lists as inputs, we cast them as Numpy arrays
										# and rebuild them afterwards if no return_tensors is specified
										# Note that we lose the specific device the tensor may be on for PyTorch

										__magic_name__						:						Optional[int]										=required_input[0]
										if isinstance(__snake_case     ,			(list, tuple)			):
														# first_element might be an empty list/tuple in some edge cases so we grab the first non empty element.
														__magic_name__						:						Optional[Any]										=0
														while len(required_input[index]			) == 0:
																		index += 1
														if index < len(__snake_case			):
																		__magic_name__						:						List[str]										=required_input[index][0]

										if return_tensors is None:
														if is_tf_tensor(__snake_case			):
																		__magic_name__						:						int										="""tf"""
														elif is_torch_tensor(__snake_case			):
																		__magic_name__						:						str										="""pt"""
														elif isinstance(__snake_case     ,			(int, float, list, tuple, np.ndarray)			):
																		__magic_name__						:						List[Any]										="""np"""
														else:
																		raise ValueError(
																		    f"type of {first_element} unknown: {type(__snake_case			)}. "
																		    """Should be one of a python, numpy, pytorch or tensorflow object."""			)

										for key, value in processed_features.items():
														if isinstance(value[0]     ,			(int, float)			):
																		__magic_name__						:						List[str]										=to_numpy(__snake_case			)
														else:
																		__magic_name__						:						str										=[to_numpy(__snake_case			) for v in value]

        # Convert padding_strategy in PaddingStrategy
										__magic_name__						:						Dict										=self._get_padding_strategies(padding=__snake_case     ,			max_length=__snake_case			)

										__magic_name__						:						Optional[Any]										=processed_features[self.model_input_names[0]]

										__magic_name__						:						Dict										=len(__snake_case			)
										if not all(len(__snake_case			) == batch_size for v in processed_features.values()			):
														raise ValueError("""Some items in the output dictionary have a different batch size than others."""			)

										__magic_name__						:						Optional[int]										=[]
										for i in range(__snake_case			):
														__magic_name__						:						Any										={k: v[i] for k, v in processed_features.items()}
														# truncation
														__magic_name__						:						List[str]										=self._truncate(
														    __snake_case     ,			max_length=__snake_case     ,			pad_to_multiple_of=__snake_case     ,			truncation=__snake_case     ,			)
														truncated_inputs.append(__snake_case			)

										if padding_strategy == PaddingStrategy.LONGEST:
														# make sure that `max_length` cannot be longer than the longest truncated length
														__magic_name__						:						Optional[int]										=max(len(input_slice[self.model_input_names[0]]			) for input_slice in truncated_inputs			)
														__magic_name__						:						Tuple										=PaddingStrategy.MAX_LENGTH

										__magic_name__						:						str										={}
										for i in range(__snake_case			):
														# padding
														__magic_name__						:						List[str]										=self._pad(
														    truncated_inputs[i]     ,			max_length=__snake_case     ,			padding_strategy=__snake_case     ,			pad_to_multiple_of=__snake_case     ,			return_attention_mask=__snake_case     ,			)

														for key, value in outputs.items():
																		if key not in batch_outputs:
																						__magic_name__						:						Dict										=[]
																		if value.dtype is np.dtype(np.floataa			):
																						__magic_name__						:						Optional[int]										=value.astype(np.floataa			)
																		batch_outputs[key].append(__snake_case			)

										return BatchFeature(__snake_case     ,			tensor_type=__snake_case			)

						def      A__    (		self :Any     ,			__snake_case :Union[Dict[str, np.ndarray], BatchFeature]     ,			__snake_case :Optional[int] = None     ,			__snake_case :PaddingStrategy = PaddingStrategy.DO_NOT_PAD     ,			__snake_case :Optional[int] = None     ,			__snake_case :Optional[bool] = None     ,			):
										'''simple docstring'''

										__magic_name__						:						Dict										=processed_features[self.model_input_names[0]]

										if padding_strategy == PaddingStrategy.LONGEST:
														__magic_name__						:						Any										=len(__snake_case			)

										if max_length is not None and pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0):
														__magic_name__						:						Dict										=((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of

										__magic_name__						:						List[Any]										=padding_strategy != PaddingStrategy.DO_NOT_PAD and len(__snake_case			) < max_length

										if return_attention_mask and "attention_mask" not in processed_features:
														__magic_name__						:						int										=np.ones(len(__snake_case			)     ,			dtype=np.intaa			)

										if needs_to_be_padded:
														__magic_name__						:						List[Any]										=max_length - len(__snake_case			)
														if self.padding_side == "right":
																		if return_attention_mask:
																						__magic_name__						:						str										=np.pad(
																						    processed_features["""attention_mask"""]     ,			(0, difference)			)
																		__magic_name__						:						Tuple										=((0, difference), (0, 0)) if self.feature_size > 1 else (0, difference)
																		__magic_name__						:						str										=np.pad(
																		    __snake_case     ,			__snake_case     ,			"""constant"""     ,			constant_values=self.padding_value			)
														elif self.padding_side == "left":
																		if return_attention_mask:
																						__magic_name__						:						str										=np.pad(
																						    processed_features["""attention_mask"""]     ,			(difference, 0)			)
																		__magic_name__						:						Optional[int]										=((difference, 0), (0, 0)) if self.feature_size > 1 else (difference, 0)
																		__magic_name__						:						List[Any]										=np.pad(
																		    __snake_case     ,			__snake_case     ,			"""constant"""     ,			constant_values=self.padding_value			)
														else:
																		raise ValueError("""Invalid padding strategy:""" + str(self.padding_side			)			)

										return processed_features

						def      A__    (		self :Optional[Any]     ,			__snake_case :Union[Dict[str, np.ndarray], BatchFeature]     ,			__snake_case :Optional[int] = None     ,			__snake_case :Optional[int] = None     ,			__snake_case :Optional[bool] = None     ,			):
										'''simple docstring'''

										if not truncation:
														return processed_features
										elif truncation and max_length is None:
														raise ValueError("""When setting ``truncation=True``, make sure that ``max_length`` is defined."""			)

										__magic_name__						:						Union[str, Any]										=processed_features[self.model_input_names[0]]

										# find `max_length` that fits `pad_to_multiple_of`
										if max_length is not None and pad_to_multiple_of is not None and (max_length % pad_to_multiple_of != 0):
														__magic_name__						:						List[str]										=((max_length // pad_to_multiple_of) + 1) * pad_to_multiple_of

										__magic_name__						:						Any										=len(__snake_case			) > max_length

										if needs_to_be_truncated:
														__magic_name__						:						List[Any]										=processed_features[self.model_input_names[0]][:max_length]
														if "attention_mask" in processed_features:
																		__magic_name__						:						List[str]										=processed_features["""attention_mask"""][:max_length]

										return processed_features


						def      A__    (		self :List[Any]     ,			__snake_case :str=False     ,			__snake_case :Optional[int]=None			):
										'''simple docstring'''

										if padding is not False:
														if padding is True:
																		__magic_name__						:						Union[str, Any]										=PaddingStrategy.LONGEST  # Default to pad to the longest sequence in the batch
														elif not isinstance(__snake_case     ,			__snake_case			):
																		__magic_name__						:						Optional[int]										=PaddingStrategy(__snake_case			)
														elif isinstance(__snake_case     ,			__snake_case			):
																		__magic_name__						:						Any										=padding
										else:
														__magic_name__						:						Any										=PaddingStrategy.DO_NOT_PAD

										# Set max length if needed
										if max_length is None:
														if padding_strategy == PaddingStrategy.MAX_LENGTH:
																		raise ValueError(
																		    f"When setting ``padding={PaddingStrategy.MAX_LENGTH}``, make sure that max_length is defined"			)

        # Test if we have a padding value
										if padding_strategy != PaddingStrategy.DO_NOT_PAD and (self.padding_value is None):
														raise ValueError(
														    """Asking to pad but the feature_extractor does not have a padding value. Please select a value to use"""
														    """ as `padding_value`. For example: `feature_extractor.padding_value = 0.0`."""			)

										return padding_strategy




