class_docstr: 实现 ``fairscale`` 功能的 ``Driver`` 。
  
  :param model: 传入给 ``Trainer`` 的 ``model`` 参数。
  :param parallel_device: 用于分布式训练的 ``gpu`` 设备。
  :param is_pull_by_torch_run: 标志当前的脚本的启动是否由 ``python -m torch.distributed.launch`` 启动的。
  :param fp16: 是否开启 fp16 训练。
  :param fairscale_kwargs:
  
      * *oss_kwargs* --
      * *sdp_kwargs* --
      * *fsdp_kwargs* --
      * *ddp_kwargs* --
      * *set_grad_to_none* -- 是否在训练过程中在每一次 optimizer 更新后将 grad 置为 ``None``
      * *non_blocking* -- 表示用于 :meth:`torch.Tensor.to` 方法的参数 non_blocking
      * *gradscaler_kwargs* -- 用于 ``fp16=True`` 时，提供给 :class:`torch.amp.cuda.GradScaler` 的参数
  :kwargs:
      * *wo_auto_param_call* (``bool``) -- 是否关闭在训练时调用我们的 ``auto_param_call`` 函数来自动匹配 batch 和前向函数的参数的行为
  
      .. note::
  
          关于该参数的详细说明，请参见 :class:`~fastNLP.core.controllers.Trainer` 中的描述；函数 ``auto_param_call`` 详见 :func:`fastNLP.core.utils.auto_param_call`。
class_docstr_tok: ["\u5b9e\u73b0","``","fairscale","``","\u529f\u80fd\u7684","``","Driver","``","\u3002",":","param","model",":","\u4f20\u5165\u7ed9","``","Trainer","``","\u7684","``","model","``","\u53c2\u6570\u3002",":","param","parallel_device",":","\u7528\u4e8e\u5206\u5e03\u5f0f\u8bad\u7ec3\u7684","``","gpu","``","\u8bbe\u5907\u3002",":","param","is_pull_by_torch_run",":","\u6807\u5fd7\u5f53\u524d\u7684\u811a\u672c\u7684\u542f\u52a8\u662f\u5426\u7531","``","python","-m","torch.distributed.launch","``","\u542f\u52a8\u7684\u3002",":","param","fp16",":","\u662f\u5426\u5f00\u542f","fp16","\u8bad\u7ec3\u3002",":","param","fairscale_kwargs",":","*","*","oss_kwargs","*","--","*","*","sdp_kwargs","*","--","*","*","fsdp_kwargs","*","--","*","*","ddp_kwargs","*","--","*","*","set_grad_to_none","*","--","\u662f\u5426\u5728\u8bad\u7ec3\u8fc7\u7a0b\u4e2d\u5728\u6bcf\u4e00\u6b21","optimizer","\u66f4\u65b0\u540e\u5c06","grad","\u7f6e\u4e3a","``","None","``","*","*","non_blocking","*","--","\u8868\u793a\u7528\u4e8e",":","meth",":","`","torch.Tensor.to","`","\u65b9\u6cd5\u7684\u53c2\u6570","non_blocking","*","*","gradscaler_kwargs","*","--","\u7528\u4e8e","``","fp16=True","``","\u65f6\uff0c\u63d0\u4f9b\u7ed9",":","class",":","`","torch.amp.cuda.GradScaler","`","\u7684\u53c2\u6570",":","kwargs",":","*","*","wo_auto_param_call","*","(","``","bool","``",")","--","\u662f\u5426\u5173\u95ed\u5728\u8bad\u7ec3\u65f6\u8c03\u7528\u6211\u4eec\u7684","``","auto_param_call","``","\u51fd\u6570\u6765\u81ea\u52a8\u5339\u914d","batch","\u548c\u524d\u5411\u51fd\u6570\u7684\u53c2\u6570\u7684\u884c\u4e3a","..","note",":",":","\u5173\u4e8e\u8be5\u53c2\u6570\u7684\u8be6\u7ec6\u8bf4\u660e\uff0c\u8bf7\u53c2\u89c1",":","class",":","`","~fastNLP.core.controllers.Trainer","`","\u4e2d\u7684\u63cf\u8ff0\uff1b\u51fd\u6570","``","auto_param_call","``","\u8be6\u89c1",":","func",":","`","fastNLP.core.utils.auto_param_call","`","\u3002"]
code: def __init__(
      self,
      model,
      parallel_device: Union[List["torch.device"], "torch.device"],
      is_pull_by_torch_run=False,
      fp16: bool = False,
      fairscale_kwargs: Dict = None,
      **kwargs,
  ):
      assert _NEED_IMPORT_FAIRSCALE, "fairscale is not imported."
      assert (
          not dist.is_initialized()
      ), "FairScaleDriver does not support initialize distributed by user."
      self._fairscale_kwargs = fairscale_kwargs
      self.fs_type = self._fairscale_kwargs.get(
          "fs_type", "sdp"
      )  # ddp, sdp, fsdp
      if self.fs_type == "fsdp":
          self._fairscale_kwargs[
              "set_grad_to_none"
          ] = self._fairscale_kwargs.get("set_grad_to_none", True)
      # 将最顶上的进行初始化
      kwargs.pop("torch_kwargs", None)
      TorchDriver.__init__(
          self,
          model=model,
          fp16=False,
          torch_kwargs=self._fairscale_kwargs,
          **kwargs,
      )
      self.is_pull_by_torch_run = is_pull_by_torch_run
      assert self.fs_type in ["ddp", "sdp", "fsdp"]
      self._oss_kwargs = self._fairscale_kwargs.get(
          "oss_kwargs", {}
      )  # 仅在 ddp 和 sdp 下有使用到
      self._sdp_kwargs = self._fairscale_kwargs.get("sdp_kwargs", {})
      self._fdsp_kwargs = self._fairscale_kwargs.get("fsdp_kwargs", {})
      self._ddp_kwargs = self._fairscale_kwargs.get("ddp_kwargs", {})
  
      if self.fs_type == "ddp" or fp16 is False:
          self.auto_cast, _grad_scaler = _build_fp16_env(dummy=not fp16)
          self.grad_scaler = _grad_scaler(
              **self._fairscale_kwargs.get("gradscaler_kwargs", {})
          )
      else:
          (
              self.auto_cast,
              self.grad_scaler,
          ) = torch.cuda.amp.autocast, ShardedGradScaler(
              **self._fairscale_kwargs.get("gradscaler_kwargs", {})
          )
  
      self.parallel_device = parallel_device
      if is_pull_by_torch_run:
          self.model_device = parallel_device
      else:
          self.model_device = parallel_device[self.local_rank]
  
      self.outside_ddp = False  # 不允许在外部初始化
      self._data_device = kwargs.get("data_device", None)
      if isinstance(self._data_device, int):
          if self._data_device < 0:
              raise ValueError(
                  "Parameter `data_device` can not be smaller than 0."
              )
          _could_use_device_num = torch.cuda.device_count()
          if self._data_device >= _could_use_device_num:
              raise ValueError(
                  "The gpu device that parameter `device` specifies is not existed."
              )
          self._data_device = torch.device(f"cuda:{self._data_device}")
      elif isinstance(self._data_device, str):
          self._data_device = torch.device(self._data_device)
      elif self._data_device is not None and not isinstance(
          self._data_device, torch.device
      ):
          raise ValueError(
              "Parameter `device` is wrong type, please check our documentation for the right use."
          )
  
      self._master_port = None
      # world_size 表示的就是全局的显卡的数量；
      self.world_size = None  # int(os.environ.get("WORLD_SIZE"))  len(self.parallel_device)
      self.global_rank = 0
  
      if self.fs_type == "ddp":
          if (
              len(self.model._buffers) != 0
              and self._ddp_kwargs.get("broadcast_buffers", None)
              is None
          ):
              logger.info(
                  "Notice your model has buffers and you are using `FairScaleDriver`, but you do not set "
                  "'broadcast_buffers' in your trainer. Cause in most situations, this parameter can be set"
                  " to 'False' to avoid redundant data communication between different processes."
              )
  
      self.output_from_new_proc = kwargs.get(
          "output_from_new_proc", "only_error"
      )
      assert isinstance(
          self.output_from_new_proc, str
      ), "Parameter `output_from_new_proc` can only be `str` type."
      if self.output_from_new_proc not in {
          "all",
          "ignore",
          "only_error",
      }:
          os.makedirs(self.output_from_new_proc, exist_ok=True)
          self.output_from_new_proc = os.path.abspath(
              self.output_from_new_proc
          )
  
      self._has_setup = (
          False  # 设置这一参数是因为 evaluator 中也会进行 setup 操作，但是显然是不需要的也不应该的；
      )
      self._has_ddpwrapped = False
  
imports: from typing import List, Sequence, Union, Dict, Mapping
  from pathlib import Path
  import os
  import functools
  from fastNLP.envs.imports import _NEED_IMPORT_FAIRSCALE
  from ...log import logger
  from .utils import _DDPWrappingModel
  from .ddp import TorchDDPDriver
  from .torch_driver import TorchDriver
  from .utils import _build_fp16_env
  from ....envs.distributed import all_rank_call_context
  from fastNLP.envs import FASTNLP_DISTRIBUTED_CHECK
  from .utils import optimizer_state_to_device
label_desc: Use image node_id 1 to create a new FairScaleDriver object from inherited base classes: TorchDDPDriver with example: obj = FairScaleDriver(model, parallel_device, is_pull_by_torch_run, fp16, fairscale_kwargs)