class_docstr: Args:
      detached_trainer_name_list: str list to get ray actor handles
      strategy:
      kl_coef: the coefficient of kl divergence loss
      sync_models_from_trainers: whether to sync models from trainers. If True, you must call sync_models_to_remote_makers() in trainers to sync models.
class_docstr_tok: ["Args",":","detached_trainer_name_list",":","str","list","to","get","ray","actor","handles","strategy",":","kl_coef",":","the","coefficient","of","kl","divergence","loss","sync_models_from_trainers",":","whether","to","sync","models","from","trainers",".","If","True",",","you","must","call","sync_models_to_remote_makers","(",")","in","trainers","to","sync","models","."]
code: def __init__(
      self,
      detached_trainer_name_list: List[str],
      strategy_fn: Callable[[], Strategy],
      # a function returns (actor, critic, reward_model, initial_model)
      model_fn: Callable[[], Tuple[Actor, Critic, RewardModel, Actor]],
      env_info: Dict[str, str] = None,
      sync_models_from_trainers: bool = False,
      buffer_cpu_offload: bool = True,
      kl_coef: float = 0.1,
      callbacks: List[MakerCallback] = [],
      eval_performance: bool = False,
      debug: bool = False,
      update_lora_weights: bool = False,
      **generate_kwargs,
  ):
      # set environment variables
      if env_info:
          set_dist_env(env_info=env_info)
      self.target_trainer_list = []
      assert len(detached_trainer_name_list) > 0
      self._detached_trainer_name_list = detached_trainer_name_list
      self.strategy = strategy_fn()
      self.buffer_cpu_offload = buffer_cpu_offload
      self.kl_coef = kl_coef
      # init models
      with self.strategy.model_init_context():
          actor, critic, reward_model, initial_model = model_fn()
      self.generate_kwargs = _set_default_generate_kwargs(
          generate_kwargs, actor
      )
      if eval_performance:
          actor_numel = get_model_numel(actor)
          critic_numel = get_model_numel(critic)
          initial_model_numel = get_model_numel(initial_model)
          reward_model_numel = get_model_numel(reward_model)
          evaluator = ExperienceMakerPerformanceEvaluator(
              actor_numel,
              critic_numel,
              initial_model_numel,
              reward_model_numel,
          )
          callbacks = callbacks + [evaluator]
  
      (
          actor,
          critic,
          reward_model,
          initial_model,
      ) = self.strategy.prepare(
          actor, critic, reward_model, initial_model
      )
      self.experience_maker = NaiveExperienceMaker(
          actor, critic, reward_model, initial_model, self.kl_coef
      )
      self.callbacks = callbacks
  
      self._model_visit_lock = Lock()
  
      self._is_fully_initialized = not sync_models_from_trainers
  
      self._debug = debug
      self._update_lora_weights = update_lora_weights
      if self._update_lora_weights:
          self.actor_lora_constructor = LoRAConstructor()
          self.critic_lora_constructor = LoRAConstructor()
  
      self.target_auto_balance = False
  
      self._target_idx = 0
  
      if self._debug:
          print(
              f"[maker{get_rank()}] will send items to {self._detached_trainer_name_list}"
          )
          if not self._is_fully_initialized:
              print(f"[maker{get_rank()}] Waiting for INIT")
  
imports: import os
  import time
  import tracemalloc
  from threading import Lock
  from typing import Any, Callable, Dict, Iterable, List, Tuple, Union
  import ray
  import torch
  from coati.experience_buffer.utils import split_experience_batch
  from coati.experience_maker import Experience, NaiveExperienceMaker
  from coati.models.base import Actor, Critic, RewardModel
  from coati.trainer.strategies import Strategy
  from torch import Tensor
  from tqdm import tqdm
  from .callbacks import ExperienceMakerPerformanceEvaluator, MakerCallback
  from .lora_constructor import LoRAConstructor
  from .utils import get_model_numel, get_rank, is_rank_0, set_dist_env, state_dict_to
label_desc: Use image node_id 1 to create a new ExperienceMakerHolder object  with example: obj = ExperienceMakerHolder(detached_trainer_name_list, strategy_fn, model_fn, env_info, sync_models_from_trainers, buffer_cpu_offload, kl_coef, callbacks, eval_performance, debug, update_lora_weights)