agent/DQ_agent.py
def _replay(self,
        s: np.ndarray,
        a: np.ndarray,
        r: np.ndarray,
        d: np.ndarray,
        s2: np.ndarray
    ) -> float:
        """
        Train the network on a mini-batch of replay data.

        Args:
            s: a batch of current states
            a: a batch of actions from each state in s
            r: a batch of reward from each action in a
            d: a batch of terminal flags after each action in a
            s2: the next state from each state, action pair in s, a

        Returns:
            the loss as a result of the training
        """
        # initialize target y values
        y = np.zeros((len(s), self.env.action_space.n))

        # predict Q values for the next state of each memory in the batch and
        # take the maximum value. dont mask any outputs, i.e. use ones
        all_mask = np.ones((len(s), self.env.action_space.n))
        Q = np.max(self.target_model.predict_on_batch([s2, all_mask]), axis=1)
        # terminal states have a Q value of zero by definition
        Q[d] = 0
        # set the y value for each sample to the reward of the selected
        # action plus the discounted Q value
        y[range(y.shape[0]), a] = r + self.discount_factor * Q

        # use an identity of size action space, and index rows from it using
        # the action vector to produce a one-hot matrix representing the mask
        action_mask = np.eye(self.env.action_space.n)[a]
        # train the model on the batch and return the loss. use the mask that
        # disables training for actions that aren't the selected actions.
        return self.model.train_on_batch([s, action_mask], y)

        Train the network on a mini-batch of replay data.

        Args:
            s: a batch of current states
            a: a batch of actions from each state in s
            r: a batch of reward from each action in a
            d: a batch of terminal flags after each action in a
            s2: the next state from each state, action pair in s, a

        Returns:
            the loss as a result of the training
        
Train the network on a mini-batch of replay data.
Train the network on a mini - batch of replay data .
def _replay(self,
        s: np.ndarray,
        a: np.ndarray,
        r: np.ndarray,
        d: np.ndarray,
        s2: np.ndarray
    ) -> float:
        y = np.zeros((len(s), self.env.action_space.n))
        all_mask = np.ones((len(s), self.env.action_space.n))
        Q = np.max(self.target_model.predict_on_batch([s2, all_mask]), axis=1)
        Q[d] = 0
        y[range(y.shape[0]), a] = r + self.discount_factor * Q
        action_mask = np.eye(self.env.action_space.n)[a]
        return self.model.train_on_batch([s, action_mask], y)
def _replay ( self , s : np . ndarray , a : np . ndarray , r : np . ndarray , d : np . ndarray , s2 : np . ndarray ) -> float : y = np . zeros ( ( len ( s ) , self . env . action_space . n ) ) all_mask = np . ones ( ( len ( s ) , self . env . action_space . n ) ) Q = np . max ( self . target_model . predict_on_batch ( [ s2 , all_mask ] ) , axis = 1 ) Q [ d ] = 0 y [ range ( y . shape [ 0 ] ) , a ] = r + self . discount_factor * Q action_mask = np . eye ( self . env . action_space . n ) [ a ] return self . model . train_on_batch ( [ s , action_mask ] , y )
Train the network on a mini-batch of replay data.
Train the network on a mini - batch of replay data .
"""
        Train the network on a mini-batch of replay data.

        Args:
            s: a batch of current states
            a: a batch of actions from each state in s
            r: a batch of reward from each action in a
            d: a batch of terminal flags after each action in a
            s2: the next state from each state, action pair in s, a

        Returns:
            the loss as a result of the training
        """ # initialize target y values # predict Q values for the next state of each memory in the batch and # take the maximum value. dont mask any outputs, i.e. use ones # terminal states have a Q value of zero by definition # set the y value for each sample to the reward of the selected # action plus the discounted Q value # use an identity of size action space, and index rows from it using # the action vector to produce a one-hot matrix representing the mask # train the model on the batch and return the loss. use the mask that # disables training for actions that aren't the selected actions.
the loss as a result of the training
the loss as a result of the training
self
s
np.ndarray
a batch of current states
a batch of current states
a
np.ndarray
a batch of actions from each state in s
a batch of actions from each state in s
r
np.ndarray
a batch of reward from each action in a
a batch of reward from each action in a
d
np.ndarray
a batch of terminal flags after each action in a
a batch of terminal flags after each action in a
s2
np.ndarray
the next state from each state, action pair in s, a
the next state from each state action pair in s a