class_docstr: Actor Critic class stores both the actor and the critic models
  and it generates values and action for given sequences during the training
  of the actor.
  
  Attributes:
      actor (ActorModel): Actor model
      critic (CriticModel): Critic model
      debug (bool): enable prints for Debugging
      use_same_tokenizer (bool): if True the actor and critic use the same
          tokenizer
  
  Methods:
      forward: given a sequence returns action logits and values (used
          to evaluate the actor during training)
      generate: given a sequence returns action, action logits, values
          sequences and sequences masks (used to generate new sequences
          during acting phase)
class_docstr_tok: ["Actor","Critic","class","stores","both","the","actor","and","the","critic","models","and","it","generates","values","and","action","for","given","sequences","during","the","training","of","the","actor",".","Attributes",":","actor","(","ActorModel",")",":","Actor","model","critic","(","CriticModel",")",":","Critic","model","debug","(","bool",")",":","enable","prints","for","Debugging","use_same_tokenizer","(","bool",")",":","if","True","the","actor","and","critic","use","the","same","tokenizer","Methods",":","forward",":","given","a","sequence","returns","action","logits","and","values","(","used","to","evaluate","the","actor","during","training",")","generate",":","given","a","sequence","returns","action",",","action","logits",",","values","sequences","and","sequences","masks","(","used","to","generate","new","sequences","during","acting","phase",")"]
docstr: Save the model to the path
  This method is implemented to save the actor model as result of RLHF
  in the folder actor_rl instead of actor.save() method that saves it
  in the actor folder.
code: def save(self) -> None:
      """Save the model to the path
      This method is implemented to save the actor model as result of RLHF
      in the folder actor_rl instead of actor.save() method that saves it
      in the actor folder.
      """
      # get the path to save the actor
      model_folder, model_name, path = ModelLoader.get_model_path(
          config=self.config,
          is_checkpoint=False,
      )
  
      # save the model
      print(f"Saving model to {path} ...")
      torch.save(
          {"state_dict": self.actor.model.state_dict()},
          path,
      )
  
      # get the path to save the critic model
      model_folder, model_name, path = ModelLoader.get_model_path(
          config=self.config.critic,
          is_checkpoint=False,
      )
  
      # save the model
      print(f"Saving model to {path} ...")
      torch.save(
          {
              "model": self.critic.model.state_dict(),
              "head": self.critic.head.state_dict(),
          },
          path,
      )
  
imports: import json
  import os
  import random
  from collections import deque, namedtuple
  import deepspeed
  import torch
  import torch.distributed
  from accelerate import Accelerator
  from beartype import beartype
  from beartype.typing import Deque, List, Tuple, Union
  from deepspeed.runtime.engine import DeepSpeedEngine
  from torch.utils.data import DataLoader, Dataset
  from torch.optim.lr_scheduler import CosineAnnealingWarmRestarts
  from chatllama.rlhf.actor import ActorModel
  from chatllama.rlhf.config import Config, ConfigActor, ConfigCritic, ConfigReward
  from chatllama.rlhf.model_list import hf_models
  from chatllama.rlhf.model_loader import ModelLoader
  from chatllama.rlhf.reward import RewardModel, CriticModel
  from chatllama.rlhf.utils import TrainingStats, ConversationLog
label_desc: Use image node_id 3 for calling the ActorCritic obj's underlying member method code with example usage: obj.save() without return types