class_docstr: Args:
      detached_trainer_name_list: str list to get ray actor handles
      strategy:
      kl_coef: the coefficient of kl divergence loss
      sync_models_from_trainers: whether to sync models from trainers. If True, you must call sync_models_to_remote_makers() in trainers to sync models.
class_docstr_tok: ["Args",":","detached_trainer_name_list",":","str","list","to","get","ray","actor","handles","strategy",":","kl_coef",":","the","coefficient","of","kl","divergence","loss","sync_models_from_trainers",":","whether","to","sync","models","from","trainers",".","If","True",",","you","must","call","sync_models_to_remote_makers","(",")","in","trainers","to","sync","models","."]
docstr: called by trainer
  chunk_start: Set True at the first call. Before sending state_dict calls
  chunk_end: Set True at the last call. After sending state_dict calls.
  fully_update: Set True if you want to sync models when initializing
  
  TODO: load_state_dict integrate with model-sharding strategy
docstr_tok: ["called","by","trainer","chunk_start",":","Set","True","at","the","first","call",".","Before","sending","state_dict","calls","chunk_end",":","Set","True","at","the","last","call",".","After","sending","state_dict","calls",".","fully_update",":","Set","True","if","you","want","to","sync","models","when","initializing","TODO",":","load_state_dict","integrate","with","model-sharding","strategy"]
code: def update_experience_maker(
      self,
      new_actor_state_dict: Dict[str, Any] = None,
      new_actor_lora_config_dict: Dict[str, Any] = None,
      new_critic_state_dict: Dict[str, Any] = None,
      new_critic_lora_config_dict: Dict[str, Any] = None,
      fully_update: bool = False,
      chunk_start: bool = None,
      chunk_end: bool = None,
  ):
      """
      called by trainer
      chunk_start: Set True at the first call. Before sending state_dict calls
      chunk_end: Set True at the last call. After sending state_dict calls.
      fully_update: Set True if you want to sync models when initializing
  
      TODO: load_state_dict integrate with model-sharding strategy
      """
      _watch_memory = self._debug
      if chunk_start:
          if self._debug:
              print("[maker] UPDATE ")
          if _watch_memory:
              tracemalloc.start()
          self._model_visit_lock.acquire()
  
      with torch.no_grad():
          if new_actor_state_dict is not None:
              if not self._update_lora_weights or fully_update:
                  self.experience_maker.actor.model.load_state_dict(
                      new_actor_state_dict, strict=False
                  )
              else:
                  new_actor_state_dict = state_dict_to(
                      new_actor_state_dict,
                      device=torch.cuda.current_device(),
                  )
                  state_dict_increase = (
                      self.actor_lora_constructor.reconstruct_increase(
                          new_actor_state_dict,
                          new_actor_lora_config_dict,
                      )
                  )
                  self.actor_lora_constructor.load_state_dict_increase(
                      self.experience_maker.actor.model,
                      state_dict_increase,
                  )
          if new_critic_state_dict is not None:
              if not self._update_lora_weights or fully_update:
                  self.experience_maker.critic.load_state_dict(
                      new_critic_state_dict, strict=False
                  )
              else:
                  new_critic_state_dict = state_dict_to(
                      new_critic_state_dict,
                      device=torch.cuda.current_device(),
                  )
                  state_dict_increase = (
                      self.critic_lora_constructor.reconstruct_increase(
                          new_critic_state_dict,
                          new_critic_lora_config_dict,
                      )
                  )
                  self.critic_lora_constructor.load_state_dict_increase(
                      self.experience_maker.critic, state_dict_increase
                  )
  
      # the lock must be released after both actor and critic being updated
      if chunk_end:
          self._model_visit_lock.release()
          if _watch_memory:
              current, peak = tracemalloc.get_traced_memory()
              print(
                  f"Current memory usage is {current / 10**6}MB; Peak was {peak / 10**6}MB"
              )
              tracemalloc.stop()
          if fully_update:
              self._is_fully_initialized = True
  
imports: import os
  import time
  import tracemalloc
  from threading import Lock
  from typing import Any, Callable, Dict, Iterable, List, Tuple, Union
  import ray
  import torch
  from coati.experience_buffer.utils import split_experience_batch
  from coati.experience_maker import Experience, NaiveExperienceMaker
  from coati.models.base import Actor, Critic, RewardModel
  from coati.trainer.strategies import Strategy
  from torch import Tensor
  from tqdm import tqdm
  from .callbacks import ExperienceMakerPerformanceEvaluator, MakerCallback
  from .lora_constructor import LoRAConstructor
  from .utils import get_model_numel, get_rank, is_rank_0, set_dist_env, state_dict_to
label_desc: Use image node_id 9 for calling the ExperienceMakerHolder obj's underlying member method code with example usage: obj.update_experience_maker(new_actor_state_dict, new_actor_lora_config_dict, new_critic_state_dict, new_critic_lora_config_dict, fully_update, chunk_start, chunk_end) without return types