class_docstr: Actor Critic class stores both the actor and the critic models
  and it generates values and action for given sequences during the training
  of the actor.
  
  Attributes:
      actor (ActorModel): Actor model
      critic (CriticModel): Critic model
      debug (bool): enable prints for Debugging
      use_same_tokenizer (bool): if True the actor and critic use the same
          tokenizer
  
  Methods:
      forward: given a sequence returns action logits and values (used
          to evaluate the actor during training)
      generate: given a sequence returns action, action logits, values
          sequences and sequences masks (used to generate new sequences
          during acting phase)
class_docstr_tok: ["Actor","Critic","class","stores","both","the","actor","and","the","critic","models","and","it","generates","values","and","action","for","given","sequences","during","the","training","of","the","actor",".","Attributes",":","actor","(","ActorModel",")",":","Actor","model","critic","(","CriticModel",")",":","Critic","model","debug","(","bool",")",":","enable","prints","for","Debugging","use_same_tokenizer","(","bool",")",":","if","True","the","actor","and","critic","use","the","same","tokenizer","Methods",":","forward",":","given","a","sequence","returns","action","logits","and","values","(","used","to","evaluate","the","actor","during","training",")","generate",":","given","a","sequence","returns","action",",","action","logits",",","values","sequences","and","sequences","masks","(","used","to","generate","new","sequences","during","acting","phase",")"]
code: def __init__(self, config: Config) -> None:
      super().__init__()
      self.config = config
  
      self.actor = ActorModel(config.actor)
  
      # check if critic must be initialized from reward model
      ModelLoader.init_critic_from_reward(config.critic)
      self.critic = CriticModel(config.critic)
  
      # if the actor and critic use the same tokenizer is set to True
      self.use_same_tokenizer = False
  
      # debug flag
      self.debug = config.actor.debug
  
imports: import json
  import os
  import random
  from collections import deque, namedtuple
  import deepspeed
  import torch
  import torch.distributed
  from accelerate import Accelerator
  from beartype import beartype
  from beartype.typing import Deque, List, Tuple, Union
  from deepspeed.runtime.engine import DeepSpeedEngine
  from torch.utils.data import DataLoader, Dataset
  from torch.optim.lr_scheduler import CosineAnnealingWarmRestarts
  from chatllama.rlhf.actor import ActorModel
  from chatllama.rlhf.config import Config, ConfigActor, ConfigCritic, ConfigReward
  from chatllama.rlhf.model_list import hf_models
  from chatllama.rlhf.model_loader import ModelLoader
  from chatllama.rlhf.reward import RewardModel, CriticModel
  from chatllama.rlhf.utils import TrainingStats, ConversationLog
label_desc: Use image node_id 1 to create a new ActorCritic object from inherited base classes: torch.nn with example: obj = ActorCritic(config)