"""simple docstring"""



import logging
import re

import pytorch_quantization
import pytorch_quantization.nn as quant_nn
import torch
from pytorch_quantization import calib
from pytorch_quantization.tensor_quant import QuantDescriptor


_UpperCamelCase								=		logging.getLogger(__name__)

_UpperCamelCase								=		50  # max width of layer names
_UpperCamelCase								=		70  # max width of quantizer names

def  _A(		lowerCAmelCase							):
							A__       :     Union[str, Any]								=			parser.add_argument_group("""quant_trainer arguments"""							)
							group.add_argument("""--wprec"""			,  type=lowerCAmelCase			,  default=8			,  help="""weight precision"""							)
							group.add_argument("""--aprec"""			,  type=lowerCAmelCase			,  default=8			,  help="""activation precision"""							)
							group.add_argument("""--quant-per-tensor"""			,  action="""store_true"""			,  help="""per tensor weight scaling"""							)
							group.add_argument("""--quant-disable"""			,  action="""store_true"""			,  help="""disable all quantizers"""							)
							group.add_argument("""--quant-disable-embeddings"""			,  action="""store_true"""			,  help="""disable all embeddings quantizers"""							)
							group.add_argument("""--quant-disable-keyword"""			,  type=lowerCAmelCase			,  nargs="""+"""			,  help="""disable quantizers by keyword"""							)
							group.add_argument("""--quant-disable-layer-module"""			,  type=lowerCAmelCase			,  help="""disable quantizers by keyword under layer."""							)
							group.add_argument("""--quant-enable-layer-module"""			,  type=lowerCAmelCase			,  help="""enable quantizers by keyword under layer"""							)
							group.add_argument("""--calibrator"""			,  default="""max"""			,  help="""which quantization range calibrator to use"""							)
							group.add_argument("""--percentile"""			,  default=lowerCAmelCase			,  type=lowerCAmelCase			,  help="""percentile for PercentileCalibrator"""							)
							group.add_argument("""--fuse-qkv"""			,  action="""store_true"""			,  help="""use the same scale factor for qkv"""							)
							group.add_argument("""--clip-gelu"""			,  metavar="""N"""			,  type=lowerCAmelCase			,  help="""clip gelu output maximum value to N"""							)
							group.add_argument(
							    """--recalibrate-weights"""			,  action="""store_true"""			,  help=(
							        """recalibrate weight amaxes by taking the max of the weights."""
							        """ amaxes will be computed with the current quantization granularity (axis)."""
							    )			,  )

def  _A(		lowerCAmelCase							):

							if args.calibrator == "max":
														A__       :     Optional[Any]								=			"""max"""
							elif args.calibrator == "percentile":
														if args.percentile is None:
																					raise ValueError("""Specify --percentile when using percentile calibrator"""							)
														A__       :     Optional[Any]								=			"""histogram"""
							elif args.calibrator == "mse":
														A__       :     int								=			"""histogram"""
							else:
														raise ValueError(F'''Invalid calibrator {args.calibrator}'''							)

							A__       :     Dict								=			QuantDescriptor(num_bits=args.aprec			,  calib_method=lowerCAmelCase							)
							A__       :     Optional[Any]								=			QuantDescriptor(num_bits=args.wprec			,  axis=(None if args.quant_per_tensor else (0,))							)
							quant_nn.QuantLinear.set_default_quant_desc_input(lowerCAmelCase							)
							quant_nn.QuantLinear.set_default_quant_desc_weight(lowerCAmelCase							)

def  _A(		lowerCAmelCase			,  lowerCAmelCase			,  lowerCAmelCase=False			,  lowerCAmelCase=False							):
							logger.info("""Configuring Model for Quantization"""							)
							logger.info(F'''using quantization package {pytorch_quantization.__file__}'''							)

							if not calib:
														if args.quant_disable_embeddings:
																					set_quantizer_by_name(lowerCAmelCase			,  ["""embeddings"""]			,  which="""weight"""			,  _disabled=lowerCAmelCase							)

														if args.quant_disable:
																					set_quantizer_by_name(lowerCAmelCase			,  [""""""]			,  _disabled=lowerCAmelCase							)

														if args.quant_disable_keyword:
																					set_quantizer_by_name(lowerCAmelCase			,  args.quant_disable_keyword			,  _disabled=lowerCAmelCase							)

														if args.quant_disable_layer_module:
																					set_quantizer_by_name(lowerCAmelCase			,  [r"""layer.\d+.""" + args.quant_disable_layer_module]			,  _disabled=lowerCAmelCase							)

														if args.quant_enable_layer_module:
																					set_quantizer_by_name(lowerCAmelCase			,  [r"""layer.\d+.""" + args.quant_enable_layer_module]			,  _disabled=lowerCAmelCase							)

														if args.recalibrate_weights:
																					recalibrate_weights(lowerCAmelCase							)

														if args.fuse_qkv:
																					fuse_qkv(lowerCAmelCase			,  lowerCAmelCase							)

							if args.clip_gelu:
														clip_gelu(lowerCAmelCase			,  args.clip_gelu							)

							# if args.local_rank in [-1, 0] and not calib:
							print_quant_summary(lowerCAmelCase							)

def  _A(		lowerCAmelCase							):
							logger.info("""Enabling Calibration"""							)
							for name, module in model.named_modules():
														if name.endswith("""_quantizer"""							):
																					if module._calibrator is not None:
																												module.disable_quant()
																												module.enable_calib()
																					else:
																												module.disable()
																					logger.info(F'''{name:80}: {module}'''							)

def  _A(		lowerCAmelCase			,  lowerCAmelCase							):
							logger.info("""Loading calibrated amax"""							)
							for name, module in model.named_modules():
														if name.endswith("""_quantizer"""							):
																					if module._calibrator is not None:
																												if isinstance(module._calibrator			,  calib.MaxCalibrator							):
																																			module.load_calib_amax()
																												else:
																																			module.load_calib_amax("""percentile"""			,  percentile=args.percentile							)
																												module.enable_quant()
																												module.disable_calib()
																					else:
																												module.enable()
							model.cuda()
							print_quant_summary(lowerCAmelCase							)

def  _A(		lowerCAmelCase			,  lowerCAmelCase							):

							def fusea(lowerCAmelCase			,  lowerCAmelCase			,  lowerCAmelCase							):
														for mod in [qq, qk, qv]:
																					if not hasattr(lowerCAmelCase			,  """_amax"""							):
																												print("""          WARNING: NO AMAX BUFFER"""							)
																												return
														A__       :     str								=			qq._amax.detach().item()
														A__       :     List[str]								=			qk._amax.detach().item()
														A__       :     str								=			qv._amax.detach().item()

														A__       :     int								=			max(lowerCAmelCase			,  lowerCAmelCase			,  lowerCAmelCase							)
														qq._amax.fill_(lowerCAmelCase							)
														qk._amax.fill_(lowerCAmelCase							)
														qv._amax.fill_(lowerCAmelCase							)
														logger.info(F'''          q={q:5.2f} k={k:5.2f} v={v:5.2f} -> {amax:5.2f}'''							)

							for name, mod in model.named_modules():
														if name.endswith(""".attention.self"""							):
																					logger.info(F'''FUSE_QKV: {name:{name_width}}'''							)
																					fusea(mod.matmul_q_input_quantizer			,  mod.matmul_k_input_quantizer			,  mod.matmul_v_input_quantizer							)
																					if args.quant_per_tensor:
																												fusea(mod.query._weight_quantizer			,  mod.key._weight_quantizer			,  mod.value._weight_quantizer							)

def  _A(		lowerCAmelCase			,  lowerCAmelCase							):

							for name, mod in model.named_modules():
														if name.endswith(""".output.dense"""							) and not name.endswith("""attention.output.dense"""							):
																					A__       :     Tuple								=			mod._input_quantizer._amax.data.detach().item()
																					mod._input_quantizer._amax.data.detach().clamp_(max=lowerCAmelCase							)
																					A__       :     List[Any]								=			mod._input_quantizer._amax.data.detach().item()
																					logger.info(F'''CLIP_GELU: {name:{name_width}} amax: {amax_init:5.2f} -> {amax:5.2f}'''							)

def  _A(		lowerCAmelCase							):

							for name, mod in model.named_modules():
														if hasattr(lowerCAmelCase			,  """_weight_quantizer"""							) and mod._weight_quantizer.axis is not None:
																					A__       :     int								=			mod.weight.shape[0]
																					A__       :     Optional[Any]								=			mod._weight_quantizer._amax.detach()
																					A__       :     Any								=			torch.ones(lowerCAmelCase			,  dtype=amax.dtype			,  device=amax.device							) * amax
																					print(F'''expanding {name} {amax} -> {mod._weight_quantizer._amax}'''							)

def  _A(		lowerCAmelCase							):

							for name, mod in model.named_modules():
														if hasattr(lowerCAmelCase			,  """_weight_quantizer"""							):
																					if not hasattr(mod.weight_quantizer			,  """_amax"""							):
																												print("""RECALIB: {name:{name_width}} WARNING: NO AMAX BUFFER"""							)
																												continue

																					# determine which axes to reduce across
																					# e.g. a 4D tensor quantized per axis 0 should reduce over (1,2,3)
																					A__       :     List[Any]								=			set() if mod._weight_quantizer.axis is None else set(mod._weight_quantizer.axis							)
																					A__       :     Union[str, Any]								=			set(range(len(mod.weight.size()							)							)							) - axis_set
																					A__       :     str								=			pytorch_quantization.utils.reduce_amax(mod.weight			,  axis=lowerCAmelCase			,  keepdims=lowerCAmelCase							).detach()
																					logger.info(F'''RECALIB: {name:{name_width}} {mod._weight_quantizer._amax.flatten()} -> {amax.flatten()}'''							)
																					A__       :     Tuple								=			amax

def  _A(		lowerCAmelCase			,  lowerCAmelCase=25			,  lowerCAmelCase=180			,  lowerCAmelCase=None							):

							if ignore is None:
														A__       :     Optional[int]								=			[]
							elif not isinstance(lowerCAmelCase			,  lowerCAmelCase							):
														A__       :     List[Any]								=			[ignore]

							A__       :     List[Any]								=			0
							for name, mod in model.named_modules():
														if not hasattr(lowerCAmelCase			,  """weight"""							):
																					continue
														A__       :     List[str]								=			max(lowerCAmelCase			,  len(lowerCAmelCase							)							)

							for name, mod in model.named_modules():
														A__       :     List[str]								=			getattr(lowerCAmelCase			,  """_input_quantizer"""			,  lowerCAmelCase							)
														A__       :     Union[str, Any]								=			getattr(lowerCAmelCase			,  """_weight_quantizer"""			,  lowerCAmelCase							)
														if not hasattr(lowerCAmelCase			,  """weight"""							):
																					continue
														if type(lowerCAmelCase							) in ignore:
																					continue
														if [True for s in ignore if type(lowerCAmelCase							) is str and s in name]:
																					continue
														A__       :     int								=			F'''Act:{input_q.extra_repr()}'''
														A__       :     Union[str, Any]								=			F'''Wgt:{weight_q.extra_repr()}'''
														A__       :     List[str]								=			F'''{name:{name_width}} {act_str} {wgt_str}'''
														if len(lowerCAmelCase							) <= line_width:
																					logger.info(lowerCAmelCase							)
														else:
																					logger.info(F'''{name:{name_width}} {act_str}'''							)
																					logger.info(F'''{"  ":{name_width}} {wgt_str}'''							)

def  _A(		lowerCAmelCase							):
							A__       :     List[str]								=			0
							for name, mod in model.named_modules():
														if isinstance(lowerCAmelCase			,  pytorch_quantization.nn.TensorQuantizer							):
																					print(F'''{name:80} {mod}'''							)
																					count += 1
							print(F'''{count} TensorQuantizers found in model'''							)

def  _A(		lowerCAmelCase			,  lowerCAmelCase			,  lowerCAmelCase			,  lowerCAmelCase			,  lowerCAmelCase							):
							A__       :     Tuple								=			getattr(lowerCAmelCase			,  lowerCAmelCase			,  lowerCAmelCase							)
							if quantizer_mod is not None:
														assert hasattr(lowerCAmelCase			,  lowerCAmelCase							)
														setattr(lowerCAmelCase			,  lowerCAmelCase			,  lowerCAmelCase							)
							else:
														logger.warning(F'''{name} has no {quantizer}'''							)

def  _A(		lowerCAmelCase			,  lowerCAmelCase			,  lowerCAmelCase="both"			,  **lowerCAmelCase							):
							A__       :     Union[str, Any]								=			F'''Warning: changing {which} quantizers of {name:{qname_width}}'''
							for k, v in kwargs.items():
														s += F''' {k}={v}'''
														if which in ["input", "both"]:
																					set_quantizer(lowerCAmelCase			,  lowerCAmelCase			,  """_input_quantizer"""			,  lowerCAmelCase			,  lowerCAmelCase							)
														if which in ["weight", "both"]:
																					set_quantizer(lowerCAmelCase			,  lowerCAmelCase			,  """_weight_quantizer"""			,  lowerCAmelCase			,  lowerCAmelCase							)
							logger.info(lowerCAmelCase							)



def  _A(		lowerCAmelCase			,  lowerCAmelCase			,  **lowerCAmelCase							):

							for name, mod in model.named_modules():
														if hasattr(lowerCAmelCase			,  """_input_quantizer"""							) or hasattr(lowerCAmelCase			,  """_weight_quantizer"""							):
																					for n in names:
																												if re.search(lowerCAmelCase			,  lowerCAmelCase							):
																																			set_quantizers(lowerCAmelCase			,  lowerCAmelCase			,  **lowerCAmelCase							)
														elif name.endswith("""_quantizer"""							):
																					for n in names:
																												if re.search(lowerCAmelCase			,  lowerCAmelCase							):
																																			A__       :     Dict								=			F'''Warning: changing {name:{name_width}}'''
																																			for k, v in kwargs.items():
																																										s += F''' {k}={v}'''
																																										setattr(lowerCAmelCase			,  lowerCAmelCase			,  lowerCAmelCase							)
																																			logger.info(lowerCAmelCase							)






"""simple docstring"""



import functools
import operator

from ...configuration_utils import PretrainedConfig
from ...utils import logging


_UpperCamelCase								=		logging.get_logger(__name__)

_UpperCamelCase								=		{
    "asapp/sew-tiny-100k": "https://huggingface.co/asapp/sew-tiny-100k/resolve/main/config.json",
    # See all SEW models at https://huggingface.co/models?filter=sew
}
class 						__UpperCAmelCase    (__A     ):







			'''simple docstring'''
			_UpperCamelCase  :						Dict					     =				'sew'





			def __init__(			self   ,      snake_case_=32   ,      snake_case_=768   ,      snake_case_=12   ,      snake_case_=12   ,      snake_case_=3_072   ,      snake_case_=2   ,      snake_case_="gelu"   ,      snake_case_=0.1   ,      snake_case_=0.1   ,      snake_case_=0.1   ,      snake_case_=0.0   ,      snake_case_=0.1   ,      snake_case_=0.1   ,      snake_case_=0.02   ,      snake_case_=1E-5   ,      snake_case_="group"   ,      snake_case_="gelu"   ,      snake_case_=(64, 128, 128, 128, 128, 256, 256, 256, 256, 512, 512, 512, 512)   ,      snake_case_=(5, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1, 2, 1)   ,      snake_case_=(10, 3, 1, 3, 1, 3, 1, 3, 1, 2, 1, 2, 1)   ,      snake_case_=False   ,      snake_case_=128   ,      snake_case_=16   ,      snake_case_=True   ,      snake_case_=0.05   ,      snake_case_=10   ,      snake_case_=2   ,      snake_case_=0.0   ,      snake_case_=10   ,      snake_case_=0   ,      snake_case_="mean"   ,      snake_case_=False   ,      snake_case_=False   ,      snake_case_=256   ,      snake_case_=0   ,      snake_case_=1   ,      snake_case_=2   ,      **snake_case_   ,      ):







										'''simple docstring'''


										super().__init__(**snake_case_   ,      pad_token_id=snake_case_   ,      bos_token_id=snake_case_   ,      eos_token_id=snake_case_							)
										A__       :     Dict								=			hidden_size
										A__       :     Dict								=			feat_extract_norm
										A__       :     str								=			feat_extract_activation
										A__       :     Optional[Any]								=			list(snake_case_							)
										A__       :     str								=			list(snake_case_							)
										A__       :     Any								=			list(snake_case_							)
										A__       :     Any								=			conv_bias
										A__       :     Any								=			num_conv_pos_embeddings
										A__       :     Any								=			num_conv_pos_embedding_groups
										A__       :     str								=			len(self.conv_dim							)
										A__       :     Tuple								=			num_hidden_layers
										A__       :     int								=			intermediate_size
										A__       :     Union[str, Any]								=			squeeze_factor
										A__       :     Union[str, Any]								=			hidden_act
										A__       :     List[str]								=			num_attention_heads
										A__       :     List[str]								=			hidden_dropout
										A__       :     Dict								=			attention_dropout
										A__       :     Tuple								=			activation_dropout
										A__       :     Optional[int]								=			feat_proj_dropout
										A__       :     Optional[Any]								=			final_dropout
										A__       :     int								=			layerdrop
										A__       :     List[Any]								=			layer_norm_eps
										A__       :     int								=			initializer_range
										A__       :     Dict								=			vocab_size

										if (
										    (len(self.conv_stride							) != self.num_feat_extract_layers)
										    or (len(self.conv_kernel							) != self.num_feat_extract_layers)
										    or (len(self.conv_dim							) != self.num_feat_extract_layers)
										):
																	raise ValueError(
																	    """Configuration for convolutional layers is incorrect."""
																	    """It is required that `len(config.conv_dim)` == `len(config.conv_stride)` == `len(config.conv_kernel)`,"""
																	    F'''but is `len(config.conv_dim) = {len(self.conv_dim							)}`, `len(config.conv_stride)'''
																	    F'''= {len(self.conv_stride							)}`, `len(config.conv_kernel) = {len(self.conv_kernel							)}`.'''							)

										# fine-tuning config parameters for SpecAugment: https://arxiv.org/abs/1904.08779
										A__       :     Optional[Any]								=			apply_spec_augment
										A__       :     int								=			mask_time_prob
										A__       :     Tuple								=			mask_time_length
										A__       :     Optional[Any]								=			mask_time_min_masks
										A__       :     Any								=			mask_feature_prob
										A__       :     List[Any]								=			mask_feature_length
										A__       :     Any								=			mask_feature_min_masks

										# ctc loss
										A__       :     str								=			ctc_loss_reduction
										A__       :     List[Any]								=			ctc_zero_infinity

										# sequence classification
										A__       :     Union[str, Any]								=			use_weighted_layer_sum
										A__       :     str								=			classifier_proj_size





			@property
			def 		lowerCamelCase					(			self							):







										'''simple docstring'''


										return functools.reduce(operator.mul   ,      self.conv_stride   ,      1							)





