class_docstr: A simple API for GPTQ Quantization
docstr: Args:
      bits (`int`):
          The number of bits to quantize to, supported numbers are (2, 3, 4, 8).
      dataset (`Union[List[str], str, Any]`, defaults to `None`):
          The dataset used for quantization. You can provide your own dataset in a list of string or in a list of tokenized data
          (e.g. [{ "input_ids": [ 1, 100, 15, ... ],"attention_mask": [ 1, 1, 1, ... ]},...])
          or just use the original datasets used in GPTQ paper ['wikitext2','c4','c4-new','ptb','ptb-new'].
      group_size (int, defaults to 128):
          The group size to use for quantization. Recommended value is 128 and -1 uses per-column quantization.
      damp_percent (`float`, defaults to `0.1`):
          The percent of the average Hessian diagonal to use for dampening, recommended value is 0.1.
      desc_act (`bool`, defaults to `False`):
          Whether to quantize columns in order of decreasing activation size.
          Setting it to False can significantly speed up inference but the perplexity may become slightly worse.
          Also known as act-order.
      sym (`bool`, defaults to `True`):
          Whether to use symetric quantization.
      true_sequential (`bool`, defaults to `True`):
          Whether to perform sequential quantization even within a single Transformer block.
          Instead of quantizing the entire block at once, we perform layer-wise quantization.
          As a result, each layer undergoes quantization using inputs that have passed through the previously quantized layers.
      use_cuda_fp16 (`bool`, defaults to `False`):
          Whether or not to use optimized cuda kernel for fp16 model. Need to have model in fp16.
      model_seqlen (`Optional[int]`, defaults to `None`):
          The maximum sequence length that the model can take.
      block_name_to_quantize (`Optional[str]`, defaults to `None`):
          The transformers block name to quantize. If None, we will infer the block name using common patterns (e.g. model.layers)
      module_name_preceding_first_block (`Optional[List[str]]`, defaults to `None`):
          The layers that are preceding the first Transformer block.
      batch_size (`int`, defaults to `1`):
          The batch size of the dataset
      pad_token_id (`Optional[int]`, defaults to `None`):
          The pad token id. Needed to prepare the dataset when `batch_size` > 1.
      disable_exllama (`bool`, defaults to `False`):
          Whether to use exllama backend. Only works with `bits` = 4.
      exllama_config (`Dict[str, Any]`, *optional*):
          The exllama config. You can specify the version of the exllama kernel through the `version` key. Defaults to `{"version": 2}` if unset.
      max_input_length (`Optional[int]`, defaults to `None`):
          The maximum input length. This is needed to initialize a buffer that depends on the maximum expected input length.
          It is specific to the exllama backend with act-order.
      cache_block_outputs (`bool`, defaults to `True`):
          Whether to cache block outputs to reuse as inputs for the succeeding block. It allows optimization of non-standard models
          (e.g. ChatGLM) but can require more time.
      modules_in_block_to_quantize (`Optional[List[List[str]]]`, defaults to `None`):
          List list of module names to quantize in the block specified. This argument is useful to exclude certain linear modules from being quantized.
          The block to quantize can be specified by setting `block_name_to_quantize`. We will quantize each list sequentially.
          If not set, we will quantize all linear layers. Example: `inside_layer_modules=[["self_attention.query_key_value"], ["mlp.dense_h_to_4h"]]`
docstr_tok: ["Args",":","bits","(","`","int","`",")",":","The","number","of","bits","to","quantize","to",",","supported","numbers","are","(","2",",","3",",","4",",","8",")",".","dataset","(","`","Union","[","List","[","str","]",",","str",",","Any","]","`",",","defaults","to","`","None","`",")",":","The","dataset","used","for","quantization",".","You","can","provide","your","own","dataset","in","a","list","of","string","or","in","a","list","of","tokenized","data","(","e.g",".","[","{","``","input_ids","''",":","[","1",",","100",",","15",",","...","]",",","''","attention_mask","''",":","[","1",",","1",",","1",",","...","]","}",",","...","]",")","or","just","use","the","original","datasets","used","in","GPTQ","paper","[","'wikitext2","'",",","'c4","'",",","'c4-new","'",",","'ptb","'",",","'ptb-new","'","]",".","group_size","(","int",",","defaults","to","128",")",":","The","group","size","to","use","for","quantization",".","Recommended","value","is","128","and","-1","uses","per-column","quantization",".","damp_percent","(","`","float","`",",","defaults","to","`","0.1","`",")",":","The","percent","of","the","average","Hessian","diagonal","to","use","for","dampening",",","recommended","value","is","0.1.","desc_act","(","`","bool","`",",","defaults","to","`","False","`",")",":","Whether","to","quantize","columns","in","order","of","decreasing","activation","size",".","Setting","it","to","False","can","significantly","speed","up","inference","but","the","perplexity","may","become","slightly","worse",".","Also","known","as","act-order",".","sym","(","`","bool","`",",","defaults","to","`","True","`",")",":","Whether","to","use","symetric","quantization",".","true_sequential","(","`","bool","`",",","defaults","to","`","True","`",")",":","Whether","to","perform","sequential","quantization","even","within","a","single","Transformer","block",".","Instead","of","quantizing","the","entire","block","at","once",",","we","perform","layer-wise","quantization",".","As","a","result",",","each","layer","undergoes","quantization","using","inputs","that","have","passed","through","the","previously","quantized","layers",".","use_cuda_fp16","(","`","bool","`",",","defaults","to","`","False","`",")",":","Whether","or","not","to","use","optimized","cuda","kernel","for","fp16","model",".","Need","to","have","model","in","fp16",".","model_seqlen","(","`","Optional","[","int","]","`",",","defaults","to","`","None","`",")",":","The","maximum","sequence","length","that","the","model","can","take",".","block_name_to_quantize","(","`","Optional","[","str","]","`",",","defaults","to","`","None","`",")",":","The","transformers","block","name","to","quantize",".","If","None",",","we","will","infer","the","block","name","using","common","patterns","(","e.g",".","model.layers",")","module_name_preceding_first_block","(","`","Optional","[","List","[","str","]","]","`",",","defaults","to","`","None","`",")",":","The","layers","that","are","preceding","the","first","Transformer","block",".","batch_size","(","`","int","`",",","defaults","to","`","1","`",")",":","The","batch","size","of","the","dataset","pad_token_id","(","`","Optional","[","int","]","`",",","defaults","to","`","None","`",")",":","The","pad","token","id",".","Needed","to","prepare","the","dataset","when","`","batch_size","`",">","1.","disable_exllama","(","`","bool","`",",","defaults","to","`","False","`",")",":","Whether","to","use","exllama","backend",".","Only","works","with","`","bits","`","=","4.","exllama_config","(","`","Dict","[","str",",","Any","]","`",",","*","optional","*",")",":","The","exllama","config",".","You","can","specify","the","version","of","the","exllama","kernel","through","the","`","version","`","key",".","Defaults","to","`","{","``","version","''",":","2","}","`","if","unset",".","max_input_length","(","`","Optional","[","int","]","`",",","defaults","to","`","None","`",")",":","The","maximum","input","length",".","This","is","needed","to","initialize","a","buffer","that","depends","on","the","maximum","expected","input","length",".","It","is","specific","to","the","exllama","backend","with","act-order",".","cache_block_outputs","(","`","bool","`",",","defaults","to","`","True","`",")",":","Whether","to","cache","block","outputs","to","reuse","as","inputs","for","the","succeeding","block",".","It","allows","optimization","of","non-standard","models","(","e.g",".","ChatGLM",")","but","can","require","more","time",".","modules_in_block_to_quantize","(","`","Optional","[","List","[","List","[","str","]","]","]","`",",","defaults","to","`","None","`",")",":","List","list","of","module","names","to","quantize","in","the","block","specified",".","This","argument","is","useful","to","exclude","certain","linear","modules","from","being","quantized",".","The","block","to","quantize","can","be","specified","by","setting","`","block_name_to_quantize","`",".","We","will","quantize","each","list","sequentially",".","If","not","set",",","we","will","quantize","all","linear","layers",".","Example",":","`","inside_layer_modules=","[","[","``","self_attention.query_key_value","''","]",",","[","``","mlp.dense_h_to_4h","''","]","]","`"]
code: def __init__(
      self,
      bits: int,
      dataset: Optional[Union[List[str], str]] = None,
      group_size: int = 128,
      damp_percent: float = 0.1,
      desc_act: bool = False,
      sym: bool = True,
      true_sequential: bool = True,
      use_cuda_fp16: bool = False,
      model_seqlen: Optional[int] = None,
      block_name_to_quantize: Optional[str] = None,
      module_name_preceding_first_block: Optional[List[str]] = None,
      batch_size: int = 1,
      pad_token_id: Optional[int] = None,
      disable_exllama: bool = False,
      exllama_config: Dict[str, Any] = None,
      max_input_length: Optional[int] = None,
      cache_block_outputs: Optional[bool] = True,
      modules_in_block_to_quantize: Optional[List[List[str]]] = None,
      *args,
      **kwargs,
  ):
      """
      Args:
          bits (`int`):
              The number of bits to quantize to, supported numbers are (2, 3, 4, 8).
          dataset (`Union[List[str], str, Any]`, defaults to `None`):
              The dataset used for quantization. You can provide your own dataset in a list of string or in a list of tokenized data
              (e.g. [{ "input_ids": [ 1, 100, 15, ... ],"attention_mask": [ 1, 1, 1, ... ]},...])
              or just use the original datasets used in GPTQ paper ['wikitext2','c4','c4-new','ptb','ptb-new'].
          group_size (int, defaults to 128):
              The group size to use for quantization. Recommended value is 128 and -1 uses per-column quantization.
          damp_percent (`float`, defaults to `0.1`):
              The percent of the average Hessian diagonal to use for dampening, recommended value is 0.1.
          desc_act (`bool`, defaults to `False`):
              Whether to quantize columns in order of decreasing activation size.
              Setting it to False can significantly speed up inference but the perplexity may become slightly worse.
              Also known as act-order.
          sym (`bool`, defaults to `True`):
              Whether to use symetric quantization.
          true_sequential (`bool`, defaults to `True`):
              Whether to perform sequential quantization even within a single Transformer block.
              Instead of quantizing the entire block at once, we perform layer-wise quantization.
              As a result, each layer undergoes quantization using inputs that have passed through the previously quantized layers.
          use_cuda_fp16 (`bool`, defaults to `False`):
              Whether or not to use optimized cuda kernel for fp16 model. Need to have model in fp16.
          model_seqlen (`Optional[int]`, defaults to `None`):
              The maximum sequence length that the model can take.
          block_name_to_quantize (`Optional[str]`, defaults to `None`):
              The transformers block name to quantize. If None, we will infer the block name using common patterns (e.g. model.layers)
          module_name_preceding_first_block (`Optional[List[str]]`, defaults to `None`):
              The layers that are preceding the first Transformer block.
          batch_size (`int`, defaults to `1`):
              The batch size of the dataset
          pad_token_id (`Optional[int]`, defaults to `None`):
              The pad token id. Needed to prepare the dataset when `batch_size` > 1.
          disable_exllama (`bool`, defaults to `False`):
              Whether to use exllama backend. Only works with `bits` = 4.
          exllama_config (`Dict[str, Any]`, *optional*):
              The exllama config. You can specify the version of the exllama kernel through the `version` key. Defaults to `{"version": 2}` if unset.
          max_input_length (`Optional[int]`, defaults to `None`):
              The maximum input length. This is needed to initialize a buffer that depends on the maximum expected input length.
              It is specific to the exllama backend with act-order.
          cache_block_outputs (`bool`, defaults to `True`):
              Whether to cache block outputs to reuse as inputs for the succeeding block. It allows optimization of non-standard models
              (e.g. ChatGLM) but can require more time.
          modules_in_block_to_quantize (`Optional[List[List[str]]]`, defaults to `None`):
              List list of module names to quantize in the block specified. This argument is useful to exclude certain linear modules from being quantized.
              The block to quantize can be specified by setting `block_name_to_quantize`. We will quantize each list sequentially.
              If not set, we will quantize all linear layers. Example: `inside_layer_modules=[["self_attention.query_key_value"], ["mlp.dense_h_to_4h"]]`
      """
  
      self.bits = bits
      self.dataset = dataset
      self.group_size = group_size
      self.damp_percent = damp_percent
      self.desc_act = desc_act
      self.sym = sym
      self.true_sequential = true_sequential
      self.use_cuda_fp16 = use_cuda_fp16
      self.model_seqlen = model_seqlen
      self.block_name_to_quantize = block_name_to_quantize
      self.module_name_preceding_first_block = (
          module_name_preceding_first_block
      )
      self.batch_size = batch_size
      self.pad_token_id = pad_token_id
      self.disable_exllama = disable_exllama
      self.exllama_config = exllama_config
      self.max_input_length = max_input_length
      self.quant_method = QuantizationMethod.GPTQ
      self.cache_block_outputs = cache_block_outputs
      self.modules_in_block_to_quantize = modules_in_block_to_quantize
  
      self.serialization_keys = [
          "bits",
          "dataset",
          "group_size",
          "damp_percent",
          "desc_act",
          "sym",
          "true_sequential",
          "quant_method",
          "modules_in_block_to_quantize",
      ]
  
      if self.bits not in [2, 3, 4, 8]:
          raise ValueError("only support quantize to [2,3,4,8] bits.")
      if self.group_size != -1 and self.group_size <= 0:
          raise ValueError(
              "group_size must be greater than 0 or equal to -1"
          )
      if not (0 < self.damp_percent < 1):
          raise ValueError("damp_percent must between 0 and 1.")
  
      if self.exllama_config is None:
          self.exllama_config = {"version": ExllamaVersion.TWO}
      else:
          if "version" not in self.exllama_config:
              raise ValueError(
                  "`exllama_config` needs to have a `version` key"
              )
          elif self.exllama_config["version"] not in [
              ExllamaVersion.ONE,
              ExllamaVersion.TWO,
          ]:
              version = self.exllama_config["version"]
              raise ValueError(
                  f"Only supported versions are in [ExllamaVersion.ONE, ExllamaVersion.TWO] - not recognized version {version}"
              )
      self.exllama_version = self.exllama_config["version"]
  
imports: import json
  import os
  from enum import Enum
  from logging import getLogger
  from typing import Any, Dict, List, Optional, Tuple, Union
  import torch
  from torch import nn
  from tqdm.auto import tqdm
  from transformers import AutoTokenizer
  from transformers.pytorch_utils import Conv1D
  from transformers.utils.quantization_config import QuantizationMethod
  from ..utils import is_accelerate_available, is_auto_gptq_available
  from ..utils.modeling_utils import recurse_getattr
  from .constants import GPTQ_CONFIG
  from .data import get_dataset, prepare_dataset
  from .utils import get_block_name_with_pattern, get_device, get_layers, get_preceding_modules, get_seqlen
label_desc: Use image node_id 1 to create a new GPTQQuantizer object from inherited base classes: object with example: obj = GPTQQuantizer(bits, dataset, group_size, damp_percent, desc_act, sym, true_sequential, use_cuda_fp16, model_seqlen, block_name_to_quantize, module_name_preceding_first_block, batch_size, pad_token_id, disable_exllama, exllama_config, max_input_length, cache_block_outputs, modules_in_block_to_quantize)