class_docstr: Base class for all optimizers.
  
  .. warning::
      Parameters need to be specified as collections that have a deterministic
      ordering that is consistent between runs. Examples of objects that don't
      satisfy those properties are sets and iterators over values of dictionaries.
  
  Args:
      params (iterable): an iterable of :class:`torch.Tensor` s or
          :class:`dict` s. Specifies what Tensors should be optimized.
      defaults: (dict): a dict containing default values of optimization
          options (used when a parameter group doesn't specify them).
class_docstr_tok: ["Base","class","for","all","optimizers",".","..","warning",":",":","Parameters","need","to","be","specified","as","collections","that","have","a","deterministic","ordering","that","is","consistent","between","runs",".","Examples","of","objects","that","don't","satisfy","those","properties","are","sets","and","iterators","over","values","of","dictionaries",".","Args",":","params","(","iterable",")",":","an","iterable","of",":","class",":","`","torch.Tensor","`","s","or",":","class",":","`","dict","`","s.","Specifies","what","Tensors","should","be","optimized",".","defaults",":","(","dict",")",":","a","dict","containing","default","values","of","optimization","options","(","used","when","a","parameter","group","does","n't","specify","them",")","."]
docstr: Resets the gradients of all optimized :class:`torch.Tensor` s.
  
  Args:
      set_to_none (bool): instead of setting to zero, set the grads to None.
          This will in general have lower memory footprint, and can modestly improve performance.
          However, it changes certain behaviors. For example:
          1. When the user tries to access a gradient and perform manual ops on it,
          a None attribute or a Tensor full of 0s will behave differently.
          2. If the user requests ``zero_grad(set_to_none=True)`` followed by a backward pass, ``.grad``\ s
          are guaranteed to be None for params that did not receive a gradient.
          3. ``torch.optim`` optimizers have a different behavior if the gradient is 0 or None
          (in one case it does the step with a gradient of 0 and in the other it skips
          the step altogether).
docstr_tok: ["Resets","the","gradients","of","all","optimized",":","class",":","`","torch.Tensor","`","s.","Args",":","set_to_none","(","bool",")",":","instead","of","setting","to","zero",",","set","the","grads","to","None",".","This","will","in","general","have","lower","memory","footprint",",","and","can","modestly","improve","performance",".","However",",","it","changes","certain","behaviors",".","For","example",":","1",".","When","the","user","tries","to","access","a","gradient","and","perform","manual","ops","on","it",",","a","None","attribute","or","a","Tensor","full","of","0s","will","behave","differently",".","2",".","If","the","user","requests","``","zero_grad","(","set_to_none=True",")","``","followed","by","a","backward","pass",",","``",".grad","``","\\","s","are","guaranteed","to","be","None","for","params","that","did","not","receive","a","gradient",".","3",".","``","torch.optim","``","optimizers","have","a","different","behavior","if","the","gradient","is","0","or","None","(","in","one","case","it","does","the","step","with","a","gradient","of","0","and","in","the","other","it","skips","the","step","altogether",")","."]
code: def zero_grad(self, set_to_none: bool = True) -> None:
      r"""Resets the gradients of all optimized :class:`torch.Tensor` s.
  
      Args:
          set_to_none (bool): instead of setting to zero, set the grads to None.
              This will in general have lower memory footprint, and can modestly improve performance.
              However, it changes certain behaviors. For example:
              1. When the user tries to access a gradient and perform manual ops on it,
              a None attribute or a Tensor full of 0s will behave differently.
              2. If the user requests ``zero_grad(set_to_none=True)`` followed by a backward pass, ``.grad``\ s
              are guaranteed to be None for params that did not receive a gradient.
              3. ``torch.optim`` optimizers have a different behavior if the gradient is 0 or None
              (in one case it does the step with a gradient of 0 and in the other it skips
              the step altogether).
      """
      foreach = self.defaults.get(
          "foreach", False
      ) or self.defaults.get("fused", False)
  
      if not hasattr(self, "_zero_grad_profile_name"):
          self._patch_step_function()
  
      per_device_and_dtype_grads: Optional[
          DefaultDict[
              torch.device, DefaultDict[torch.dtype, List[torch.Tensor]]
          ]
      ]
      if foreach:
          per_device_and_dtype_grads = defaultdict(
              lambda: defaultdict(list)
          )
      else:
          per_device_and_dtype_grads = None
  
      with torch.autograd.profiler.record_function(
          self._zero_grad_profile_name
      ):
          for group in self.param_groups:
              for p in group["params"]:
                  if p.grad is not None:
                      if set_to_none:
                          p.grad = None
                      else:
                          if p.grad.grad_fn is not None:
                              p.grad.detach_()
                          else:
                              p.grad.requires_grad_(False)
                          if not foreach or p.grad.is_sparse:
                              p.grad.zero_()
                          else:
                              assert (
                                  per_device_and_dtype_grads is not None
                              )
                              per_device_and_dtype_grads[p.grad.device][
                                  p.grad.dtype
                              ].append(p.grad)
          if foreach:
              assert per_device_and_dtype_grads is not None
              for (
                  per_dtype_grads
              ) in per_device_and_dtype_grads.values():
                  for grads in per_dtype_grads.values():
                      torch._foreach_zero_(grads)
  
imports: import math
  import functools
  import warnings
  from collections import OrderedDict, defaultdict
  from copy import deepcopy
  from itertools import chain
  from typing import Any, Callable, DefaultDict, Dict, Hashable, Iterable, List, Optional, Set, Tuple, TypeVar, Union, cast, overload
  from typing_extensions import ParamSpec, Self, TypeAlias
  import torch
  import torch.utils.hooks
  from torch.utils.hooks import RemovableHandle
  from torch.utils._foreach_utils import Indices, TensorListList, _get_foreach_kernels_supported_devices, _get_fused_kernels_supported_devices
  from torch._utils import is_compiling
  from torch.utils._foreach_utils import _group_tensors_by_device_and_dtype
label_desc: Use image node_id 19 for calling the Optimizer obj's underlying member method code with example usage: obj.zero_grad(set_to_none) without return types