class_docstr: Controller.`
  Keep Track of each ship and shipyard
docstr: Update the state of each tracked ship
     Define reward and keep tracking of each ship
  Args:
      board (envs.halite.helpers.Board): the environment
      alive : the status of current player
      env_done : the status of the environment
docstr_tok: ["Update","the","state","of","each","tracked","ship","Define","reward","and","keep","tracking","of","each","ship","Args",":","board","(","envs.halite.helpers.Board",")",":","the","environment","alive",":","the","status","of","current","player","env_done",":","the","status","of","the","environment"]
code: def update_ship_state(self, board, alive, env_done):
      """Update the state of each tracked ship
         Define reward and keep tracking of each ship
      Args:
          board (envs.halite.helpers.Board): the environment
          alive : the status of current player
          env_done : the status of the environment
      """
  
      old_board = self.board
      new_board = board
  
      old_me = old_board.current_player
      new_me = board.current_player
  
      for ship_id, ship_state in self.ship_states.items():
          # ship loss
          if ship_id not in new_me.ship_ids:
              # simply set this flag to a negative number
              self.ship_states[ship_id]["halite"].append(-1)
  
              # whether it's rule policy
              if ship_state["rule"]:
                  act = ship_state["act"]
  
                  # set the terminal to be 1 and this means the agent finishes its episode
                  if act in ["mine", "convert", "return_to_base"]:
                      ship_state["terminal"].append(1)
  
                  continue
  
              # Attacked by the opponents or ships in current teams
              if ship_state["act"][-1] in [0, 1, 2, 3, 4]:
                  # rew for every time step
                  rew = -1
  
                  # rew for being destroyed
                  rew += -halite
  
                  ship_state["rew"].append(rew)
  
              # Set the flags
              ship_state["value"].append(0)
              ship_state["terminal"].append(1)
              ship_state["full_episode"] = True
  
          else:
              old_ship = old_board.ships[ship_id]
  
              new_ship = board.ships[ship_id]
  
              # record the halite of this ship
              self.ship_states[ship_id]["halite"].append(
                  new_ship.halite
              )
  
              # whether it's controlled by rules
              # define the status of these rule ships
              if ship_state["rule"]:
                  if ship_state["act"] == "mine":
                      # The agent will only stop mining until the number of halite is more than 70
                      # when it's asked to mine and build a shipyard
                      if new_ship.halite > 70:
                          ship_state["terminal"].append(1)
  
                  if ship_state["act"] == "return_to_base":
                      # To determine whether the ship is at the shipyard or not
                      new_cell = board.cells[new_ship.position]
  
                      if (
                          new_cell.shipyard_id
                          and new_cell.shipyard.player_id
                          == new_ship.player_id
                      ):
                          ship_state["terminal"].append(1)
  
                  continue
  
              # To determine whether the ship should return to base or not
              return_to_base = False
              shipyard_pos = nearest_shipyard_position(board, new_ship)
              if board.cells[shipyard_pos].shipyard_id:
                  dis = board.step + mahattan_distance(
                      shipyard_pos, new_ship.position
                  )
                  return_to_base = dis <= 299 and dis >= 294
  
              if ship_state["act"][-1] in [0, 1, 2, 3, 4]:
                  # reward for every time step
                  rew = -1
  
                  # mining halite
                  if ship_state["act"][-1] == 0:
                      old_cell = old_board.cells[old_ship.position]
                      new_cell = board.cells[new_ship.position]
                      delta_cell_halite = (
                          old_cell.halite - new_cell.halite
                      )
                  else:
                      delta_cell_halite = 0
  
                  # rew for destroying other ships
                  if (
                      new_ship.halite
                      != old_ship.halite + delta_cell_halite
                  ):
                      destroyed_ship_halite = new_ship.halite - (
                          old_ship.halite + delta_cell_halite
                      )
                      rew += destroyed_ship_halite / 2
  
                  # this ships controlled by model collect the halite we need
                  if new_ship.halite >= halite:
                      ship_state["rew"].append(rew)
                      ship_state["terminal"].append(1)
                      ship_state["value"].append(0)
                      ship_state["full_episode"] = True
  
                  else:
                      # the environment ends(the enemy dies)
                      if env_done or return_to_base:
                          rew += ship_state["halite"][-1] - halite
                          ship_state["rew"].append(rew)
                          next_obs = get_ship_feature(board, new_ship)
                          ship_state["value"].append(
                              self.ship_agent.value(next_obs).squeeze(
                                  1
                              )[0]
                          )
                          ship_state["full_episode"] = True
                          ship_state["terminal"].append(0)
                      else:
                          ship_state["rew"].append(rew)
                          ship_state["terminal"].append(0)
  
      # record the id of the ship we want to loss track of
      eliminated_ship_ids = []
  
      for ship_id, ship_state in self.ship_states.items():
          if ship_state["rule"]:
              # if the rule agent finish its goal, then we should not keep track of this agent.
              if (
                  len(ship_state["terminal"])
                  and ship_state["terminal"][-1]
              ):
                  eliminated_ship_ids.append(ship_id)
  
              continue
  
          if ship_state["full_episode"]:
              # prepare data for training
              if self.train:
                  obs_batch = np.concatenate(ship_state["obs"])
                  action_batch = np.array(ship_state["act"])
                  reward_batch = np.array(ship_state["rew"])
                  terminal_batch = 1 - np.array(ship_state["terminal"])
                  log_prob_batch = np.array(ship_state["log_prob"])
                  value_batch = np.array(ship_state["value"])
  
                  return_batch = self.compute_returns(
                      reward_batch,
                      terminal_batch,
                      value_batch[-1],
                      config["gamma"],
                  )
                  value_batch = value_batch[:-1]
  
                  advantages = (
                      return_batch - terminal_batch * value_batch
                  )
                  return_batch = (
                      return_batch - return_batch.mean()
                  ) / (return_batch.std() + 1e-5)
                  advantages = (advantages - advantages.mean()) / (
                      advantages.std() + 1e-5
                  )
  
                  # add data into buffer
                  self.ship_buffer.append(
                      obs_batch,
                      action_batch,
                      value_batch,
                      return_batch,
                      log_prob_batch,
                      advantages,
                  )
  
                  # save statistic data
                  self.ship_rew.append(sum(reward_batch))
  
                  # if the number of halite collected by this ship is greater than defined number,
                  # then we record the number of steps for analysing
                  if ship_state["halite"][-1] > halite:
                      self.ship_len.append(len(action_batch))
  
              eliminated_ship_ids.append(ship_id)
  
      # loss track of eliminated ships
      for ship_id in eliminated_ship_ids:
          del self.ship_states[ship_id]
  
      # add new ship id (keep track of a new ship)
      for ship_id in new_me.ship_ids:
          if ship_id not in self.ship_states.keys():
              self.ship_states[ship_id] = {
                  "obs": [],
                  "act": [],
                  "rew": [],
                  "terminal": [],
                  "value": [],
                  "log_prob": [],
                  "rule": False,
                  "full_episode": False,
                  "halite": [],
              }
  
imports: import numpy
  from config import config
  from zerosum_env.envs.halite.helpers import *
  from .agent import Agent
  from .model import Model
  from .algorithm import PPO
  from .obs_parser import get_ship_feature
  from .replay_memory import ReplayMemory
  from .utils import is_alive, check_nearby_ship, nearest_shipyard_position, mahattan_distance
  from .policy import do_nothing_policy, move_up_policy, move_down_policy, move_left_policy, move_right_policy, spawn_policy, return_to_base_policy, mine_policy, convert_policy
label_desc: Use image node_id 10 for calling the Controller obj's underlying member method code with example usage: obj.update_ship_state(board, alive, env_done) without return types