ravens/agents/transporter.py
def act(self, obs, info, debug_imgs=False, goal=None):
        """Run inference and return best action given visual observations.

        If goal-conditioning, provide `goal`. Both `obs` and `goal` have
        'color' and 'depth' keys, but `obs['color']` and `goal['color']` are
        of type list and np.array, respectively. This is different from
        training, above, where both `obs` and `goal` are sampled from the
        dataset class, which will load both as np.arrays. Here, the `goal` is
        still from dataset, but `obs` is from the environment stepping, which
        returns things in a list. Wrap an np.array(...) to get shapes:

        np.array(obs['color']) and goal['color']: (3, 480, 640, 3)
        np.array(obs['depth']) and goal['depth']: (3, 480, 640)
        """
        act = {'camera_config': self.camera_config, 'primitive': None}
        if not obs:
            return act

        # Get heightmap from RGB-D images.
        colormap, heightmap = self.get_heightmap(obs, self.camera_config)
        if goal is not None:
            colormap_g, heightmap_g = self.get_heightmap(goal, self.camera_config)

        # Concatenate color with depth images.
        input_image = self.concatenate_c_h(colormap, heightmap)

        # Make a goal image if needed, and for consistency stack with input.
        if self.use_goal_image:
            goal_image = self.concatenate_c_h(colormap_g, heightmap_g)
            input_image = np.concatenate((input_image, goal_image), axis=2)
            assert input_image.shape[2] == 12, input_image.shape

        # Attention model forward pass.
        if self.attn_no_targ and self.use_goal_image:
            maxdim = int(input_image.shape[2] / 2)
            input_only = input_image[:, :, :maxdim]
            attention = self.attention_model.forward(input_only)
        else:
            attention = self.attention_model.forward(input_image)
        argmax = np.argmax(attention)
        argmax = np.unravel_index(argmax, shape=attention.shape)
        p0_pixel = argmax[:2]
        p0_theta = argmax[2] * (2 * np.pi / attention.shape[2])

        # Transport model forward pass.
        if isinstance(self.transport_model, TransportGoal):
            half = int(input_image.shape[2] / 2)
            img_curr = input_image[:, :, :half]
            img_goal = input_image[:, :, half:]
            transport = self.transport_model.forward(img_curr, img_goal, p0_pixel)
        else:
            transport = self.transport_model.forward(input_image, p0_pixel)
        argmax = np.argmax(transport)
        argmax = np.unravel_index(argmax, shape=transport.shape)
        p1_pixel = argmax[:2]
        p1_theta = argmax[2] * (2 * np.pi / transport.shape[2])

        # Pixels to end effector poses.
        p0_position = utils.pixel_to_position(p0_pixel, heightmap, self.bounds, self.pixel_size)
        p1_position = utils.pixel_to_position(p1_pixel, heightmap, self.bounds, self.pixel_size)
        p0_rotation = p.getQuaternionFromEuler((0, 0, -p0_theta))
        p1_rotation = p.getQuaternionFromEuler((0, 0, -p1_theta))

        act['primitive'] = 'pick_place'
        if self.task == 'sweeping':
            act['primitive'] = 'sweep'
        elif self.task == 'pushing':
            act['primitive'] = 'push'
        params = {'pose0': (p0_position, p0_rotation),
                  'pose1': (p1_position, p1_rotation)}
        act['params'] = params

        # Daniel: determine the task stage if applicable. (AND if loading only)
        if self.task in ['bag-items-easy', 'bag-items-hard', 'bag-color-goal']:
            self._determine_task_stage(p0_pixel, p1_pixel)

        # Daniel: only change is potentially returning more info.
        if debug_imgs:
            # FWIW, attention (320,160,1), and already has softmax applied.
            # Then the attention heat map will return a (160,320,3) image.
            # The transport also has softmax, and is shape (320,160,num_rot).
            # Thus, t_heat is actually a LIST of (160,320,3) shaped images.
            # (For forward passes, we apply the softmax to the attention and
            # transport tensors; for training we don't because the TensorFlow
            # cross entropy loss assumes it's applied before the softmax.)
            a_heat = self.attention_model.get_attention_heatmap(attention)
            t_heat = self.transport_model.get_transport_heatmap(transport)
            extras = {
                'input_c': cv2.cvtColor(colormap, cv2.COLOR_RGB2BGR),
                'attn_heat_bgr': a_heat,  # already converted to BGR
                'tran_heat_bgr': t_heat,  # already converted to BGR
                'tran_rot_argmax': argmax[2],
                'tran_p1_theta': p1_theta,
            }
            # Images by default should be vertically oriented. Can make
            # horizontal if we use .transpose(1,0,2).
            return act, extras
        else:
            return act
Run inference and return best action given visual observations.

        If goal-conditioning, provide `goal`. Both `obs` and `goal` have
        'color' and 'depth' keys, but `obs['color']` and `goal['color']` are
        of type list and np.array, respectively. This is different from
        training, above, where both `obs` and `goal` are sampled from the
        dataset class, which will load both as np.arrays. Here, the `goal` is
        still from dataset, but `obs` is from the environment stepping, which
        returns things in a list. Wrap an np.array(...) to get shapes:

        np.array(obs['color']) and goal['color']: (3, 480, 640, 3)
        np.array(obs['depth']) and goal['depth']: (3, 480, 640)
        
Run inference and return best action given visual observations.
If goal-conditioning, provide `goal`.


Run inference and return best action given visual observations . If goal - conditioning provide ` goal ` .
def act(self, obs, info, debug_imgs=False, goal=None):
        act = {'camera_config': self.camera_config, 'primitive': None}
        if not obs:
            return act
        colormap, heightmap = self.get_heightmap(obs, self.camera_config)
        if goal is not None:
            colormap_g, heightmap_g = self.get_heightmap(goal, self.camera_config)
        input_image = self.concatenate_c_h(colormap, heightmap)
        if self.use_goal_image:
            goal_image = self.concatenate_c_h(colormap_g, heightmap_g)
            input_image = np.concatenate((input_image, goal_image), axis=2)
            assert input_image.shape[2] == 12, input_image.shape
        if self.attn_no_targ and self.use_goal_image:
            maxdim = int(input_image.shape[2] / 2)
            input_only = input_image[:, :, :maxdim]
            attention = self.attention_model.forward(input_only)
        else:
            attention = self.attention_model.forward(input_image)
        argmax = np.argmax(attention)
        argmax = np.unravel_index(argmax, shape=attention.shape)
        p0_pixel = argmax[:2]
        p0_theta = argmax[2] * (2 * np.pi / attention.shape[2])
        if isinstance(self.transport_model, TransportGoal):
            half = int(input_image.shape[2] / 2)
            img_curr = input_image[:, :, :half]
            img_goal = input_image[:, :, half:]
            transport = self.transport_model.forward(img_curr, img_goal, p0_pixel)
        else:
            transport = self.transport_model.forward(input_image, p0_pixel)
        argmax = np.argmax(transport)
        argmax = np.unravel_index(argmax, shape=transport.shape)
        p1_pixel = argmax[:2]
        p1_theta = argmax[2] * (2 * np.pi / transport.shape[2])
        p0_position = utils.pixel_to_position(p0_pixel, heightmap, self.bounds, self.pixel_size)
        p1_position = utils.pixel_to_position(p1_pixel, heightmap, self.bounds, self.pixel_size)
        p0_rotation = p.getQuaternionFromEuler((0, 0, -p0_theta))
        p1_rotation = p.getQuaternionFromEuler((0, 0, -p1_theta))
        act['primitive'] = 'pick_place'
        if self.task == 'sweeping':
            act['primitive'] = 'sweep'
        elif self.task == 'pushing':
            act['primitive'] = 'push'
        params = {'pose0': (p0_position, p0_rotation),
                  'pose1': (p1_position, p1_rotation)}
        act['params'] = params
        if self.task in ['bag-items-easy', 'bag-items-hard', 'bag-color-goal']:
            self._determine_task_stage(p0_pixel, p1_pixel)
        if debug_imgs:
            a_heat = self.attention_model.get_attention_heatmap(attention)
            t_heat = self.transport_model.get_transport_heatmap(transport)
            extras = {
                'input_c': cv2.cvtColor(colormap, cv2.COLOR_RGB2BGR),
                'attn_heat_bgr': a_heat,  
                'tran_heat_bgr': t_heat,  
                'tran_rot_argmax': argmax[2],
                'tran_p1_theta': p1_theta,
            }
            return act, extras
        else:
            return act
def act ( self , obs , info , debug_imgs = False , goal = None ) : act = { 'camera_config' : self . camera_config , 'primitive' : None } if not obs : return act colormap , heightmap = self . get_heightmap ( obs , self . camera_config ) if goal is not None : colormap_g , heightmap_g = self . get_heightmap ( goal , self . camera_config ) input_image = self . concatenate_c_h ( colormap , heightmap ) if self . use_goal_image : goal_image = self . concatenate_c_h ( colormap_g , heightmap_g ) input_image = np . concatenate ( ( input_image , goal_image ) , axis = 2 ) assert input_image . shape [ 2 ] == 12 , input_image . shape if self . attn_no_targ and self . use_goal_image : maxdim = int ( input_image . shape [ 2 ] / 2 ) input_only = input_image [ : , : , : maxdim ] attention = self . attention_model . forward ( input_only ) else : attention = self . attention_model . forward ( input_image ) argmax = np . argmax ( attention ) argmax = np . unravel_index ( argmax , shape = attention . shape ) p0_pixel = argmax [ : 2 ] p0_theta = argmax [ 2 ] * ( 2 * np . pi / attention . shape [ 2 ] ) if isinstance ( self . transport_model , TransportGoal ) : half = int ( input_image . shape [ 2 ] / 2 ) img_curr = input_image [ : , : , : half ] img_goal = input_image [ : , : , half : ] transport = self . transport_model . forward ( img_curr , img_goal , p0_pixel ) else : transport = self . transport_model . forward ( input_image , p0_pixel ) argmax = np . argmax ( transport ) argmax = np . unravel_index ( argmax , shape = transport . shape ) p1_pixel = argmax [ : 2 ] p1_theta = argmax [ 2 ] * ( 2 * np . pi / transport . shape [ 2 ] ) p0_position = utils . pixel_to_position ( p0_pixel , heightmap , self . bounds , self . pixel_size ) p1_position = utils . pixel_to_position ( p1_pixel , heightmap , self . bounds , self . pixel_size ) p0_rotation = p . getQuaternionFromEuler ( ( 0 , 0 , - p0_theta ) ) p1_rotation = p . getQuaternionFromEuler ( ( 0 , 0 , - p1_theta ) ) act [ 'primitive' ] = 'pick_place' if self . task == 'sweeping' : act [ 'primitive' ] = 'sweep' elif self . task == 'pushing' : act [ 'primitive' ] = 'push' params = { 'pose0' : ( p0_position , p0_rotation ) , 'pose1' : ( p1_position , p1_rotation ) } act [ 'params' ] = params if self . task in [ 'bag-items-easy' , 'bag-items-hard' , 'bag-color-goal' ] : self . _determine_task_stage ( p0_pixel , p1_pixel ) if debug_imgs : a_heat = self . attention_model . get_attention_heatmap ( attention ) t_heat = self . transport_model . get_transport_heatmap ( transport ) extras = { 'input_c' : cv2 . cvtColor ( colormap , cv2 . COLOR_RGB2BGR ) , 'attn_heat_bgr' : a_heat , 'tran_heat_bgr' : t_heat , 'tran_rot_argmax' : argmax [ 2 ] , 'tran_p1_theta' : p1_theta , } return act , extras else : return act
Run inference and return best action given visual observations.
Run inference and return best action given visual observations .
"""Run inference and return best action given visual observations.

        If goal-conditioning, provide `goal`. Both `obs` and `goal` have
        'color' and 'depth' keys, but `obs['color']` and `goal['color']` are
        of type list and np.array, respectively. This is different from
        training, above, where both `obs` and `goal` are sampled from the
        dataset class, which will load both as np.arrays. Here, the `goal` is
        still from dataset, but `obs` is from the environment stepping, which
        returns things in a list. Wrap an np.array(...) to get shapes:

        np.array(obs['color']) and goal['color']: (3, 480, 640, 3)
        np.array(obs['depth']) and goal['depth']: (3, 480, 640)
        """ # Get heightmap from RGB-D images. # Concatenate color with depth images. # Make a goal image if needed, and for consistency stack with input. # Attention model forward pass. # Transport model forward pass. # Pixels to end effector poses. # Daniel: determine the task stage if applicable. (AND if loading only) # Daniel: only change is potentially returning more info. # FWIW, attention (320,160,1), and already has softmax applied. # Then the attention heat map will return a (160,320,3) image. # The transport also has softmax, and is shape (320,160,num_rot). # Thus, t_heat is actually a LIST of (160,320,3) shaped images. # (For forward passes, we apply the softmax to the attention and # transport tensors; for training we don't because the TensorFlow # cross entropy loss assumes it's applied before the softmax.) # already converted to BGR # already converted to BGR # Images by default should be vertically oriented. Can make # horizontal if we use .transpose(1,0,2).
self
obs
info
debug_imgs
goal