models/scm_model.py
def forward(self, input_dict, state, seq_lens):
        """
        The forward pass through the SCM network.
        :param input_dict: The input tensors.
        :param state: The model state.
        :param seq_lens: The LSTM sequence lengths.
        :return: The SCM output and new model state.
        """
        output, new_state = super(SocialCuriosityModule, self).forward(input_dict, state, seq_lens)

        encoded_state = self.scm_encoder_model(input_dict["obs"]["curr_obs"])
        new_state.append(encoded_state)

        influence_reward = tf.expand_dims(self._social_influence_reward, axis=-1)
        one_hot_actions = tf.reshape(
            self._true_one_hot_actions, shape=[-1, self._true_one_hot_actions.shape[-1]]
        )
        # Stop backpropagation through the LSTM.
        # This is done because the SCM should not influence what the MOA is modeling
        lstm_input = tf.stop_gradient(state[2])

        # TODO(@internetcoffeephone): Change state[6] magic number to something that does not depend
        #  on the order
        # Note that the inputs are different from the paper: we can only work with historical actions
        # values, so the inputs of the forward and inverse models are "behind" by 1 timestep, which
        # is corrected for in the reward function.
        forward_model_input = {
            # Encoded state at t-1
            "encoded_input_now": state[6],
            # Social influence at t-1
            "influence_reward_input": influence_reward,
            # Actions at t-1
            "action_input": one_hot_actions,
            # MOA LSTM output at t-1
            "lstm_input": lstm_input,
        }

        inverse_model_input = {
            # Encoded state at t-1
            "encoded_input_now": state[6],
            # Encoded state at t
            "encoded_input_next": encoded_state,
            # Actions at t-1
            "action_input": one_hot_actions,
            # MOA LSTM output at t-1
            "lstm_input": lstm_input,
        }

        forward_model_output = self.forward_model(forward_model_input)
        inverse_model_output = self.inverse_model(inverse_model_input)

        curiosity_reward = self.compute_curiosity_reward(encoded_state, forward_model_output)
        curiosity_reward = tf.reshape(curiosity_reward, [-1])
        self._social_curiosity_reward = curiosity_reward

        inverse_model_loss = self.compute_inverse_model_loss(influence_reward, inverse_model_output)
        self._inverse_model_loss = tf.reshape(inverse_model_loss, [-1])

        return output, new_state

        The forward pass through the SCM network.
        :param input_dict: The input tensors.
        :param state: The model state.
        :param seq_lens: The LSTM sequence lengths.
        :return: The SCM output and new model state.
        
The forward pass through the SCM network.
The forward pass through the SCM network .
def forward(self, input_dict, state, seq_lens):
        output, new_state = super(SocialCuriosityModule, self).forward(input_dict, state, seq_lens)
        encoded_state = self.scm_encoder_model(input_dict["obs"]["curr_obs"])
        new_state.append(encoded_state)
        influence_reward = tf.expand_dims(self._social_influence_reward, axis=-1)
        one_hot_actions = tf.reshape(
            self._true_one_hot_actions, shape=[-1, self._true_one_hot_actions.shape[-1]]
        )
        lstm_input = tf.stop_gradient(state[2])
        forward_model_input = {
            "encoded_input_now": state[6],
            "influence_reward_input": influence_reward,
            "action_input": one_hot_actions,
            "lstm_input": lstm_input,
        }
        inverse_model_input = {
            "encoded_input_now": state[6],
            "encoded_input_next": encoded_state,
            "action_input": one_hot_actions,
            "lstm_input": lstm_input,
        }
        forward_model_output = self.forward_model(forward_model_input)
        inverse_model_output = self.inverse_model(inverse_model_input)
        curiosity_reward = self.compute_curiosity_reward(encoded_state, forward_model_output)
        curiosity_reward = tf.reshape(curiosity_reward, [-1])
        self._social_curiosity_reward = curiosity_reward
        inverse_model_loss = self.compute_inverse_model_loss(influence_reward, inverse_model_output)
        self._inverse_model_loss = tf.reshape(inverse_model_loss, [-1])
        return output, new_state
def forward ( self , input_dict , state , seq_lens ) : output , new_state = super ( SocialCuriosityModule , self ) . forward ( input_dict , state , seq_lens ) encoded_state = self . scm_encoder_model ( input_dict [ "obs" ] [ "curr_obs" ] ) new_state . append ( encoded_state ) influence_reward = tf . expand_dims ( self . _social_influence_reward , axis = - 1 ) one_hot_actions = tf . reshape ( self . _true_one_hot_actions , shape = [ - 1 , self . _true_one_hot_actions . shape [ - 1 ] ] ) lstm_input = tf . stop_gradient ( state [ 2 ] ) forward_model_input = { "encoded_input_now" : state [ 6 ] , "influence_reward_input" : influence_reward , "action_input" : one_hot_actions , "lstm_input" : lstm_input , } inverse_model_input = { "encoded_input_now" : state [ 6 ] , "encoded_input_next" : encoded_state , "action_input" : one_hot_actions , "lstm_input" : lstm_input , } forward_model_output = self . forward_model ( forward_model_input ) inverse_model_output = self . inverse_model ( inverse_model_input ) curiosity_reward = self . compute_curiosity_reward ( encoded_state , forward_model_output ) curiosity_reward = tf . reshape ( curiosity_reward , [ - 1 ] ) self . _social_curiosity_reward = curiosity_reward inverse_model_loss = self . compute_inverse_model_loss ( influence_reward , inverse_model_output ) self . _inverse_model_loss = tf . reshape ( inverse_model_loss , [ - 1 ] ) return output , new_state
The forward pass through the SCM network.
The forward pass through the SCM network .
"""
        The forward pass through the SCM network.
        :param input_dict: The input tensors.
        :param state: The model state.
        :param seq_lens: The LSTM sequence lengths.
        :return: The SCM output and new model state.
        """ # Stop backpropagation through the LSTM. # This is done because the SCM should not influence what the MOA is modeling # TODO(@internetcoffeephone): Change state[6] magic number to something that does not depend #  on the order # Note that the inputs are different from the paper: we can only work with historical actions # values, so the inputs of the forward and inverse models are "behind" by 1 timestep, which # is corrected for in the reward function. # Encoded state at t-1 # Social influence at t-1 # Actions at t-1 # MOA LSTM output at t-1 # Encoded state at t-1 # Encoded state at t # Actions at t-1 # MOA LSTM output at t-1
The SCM output and new model state.
The SCM output and new model state .
self
input_dict
The input tensors.
The input tensors .
state
The model state.
The model state .
seq_lens
The LSTM sequence lengths.
The LSTM sequence lengths .