ravens/models/transport_goal.py
def forward(self, in_img, goal_img, p, apply_softmax=True):
        """Forward pass of our goal-conditioned Transporter.

        Relevant shapes and info:

            in_img and goal_img: (320,160,6)
            p: integer pixels on in_img, e.g., [158, 30]
            self.padding: [[32,32],[32,32],0,0]], with shape (3,2)

        Run input through all three networks, to get output of the same
        shape, except that the last channel is 3 (output_dim). Then, the
        output for one stream has the convolutional kernels for another. Call
        tf.nn.convolution. That's it, and the operation is be differentiable,
        so that gradients apply to all the FCNs.

        I actually think cropping after the query network is easier, because
        otherwise we have to do a forward pass, then call tf.multiply, then
        do another forward pass, which splits up the computation.
        """
        assert in_img.shape == goal_img.shape, f'{in_img.shape}, {goal_img.shape}'

        # input image --> TF tensor
        input_unproc = np.pad(in_img, self.padding, mode='constant')    # (384,224,6)
        input_data = self.preprocess(input_unproc.copy())               # (384,224,6)
        input_shape = (1,) + input_data.shape
        input_data = input_data.reshape(input_shape)                    # (1,384,224,6)
        in_tensor = tf.convert_to_tensor(input_data, dtype=tf.float32)  # (1,384,224,6)

        # goal image --> TF tensor
        goal_unproc = np.pad(goal_img, self.padding, mode='constant')   # (384,224,6)
        goal_data = self.preprocess(goal_unproc.copy())                 # (384,224,6)
        goal_shape = (1,) + goal_data.shape
        goal_data = goal_data.reshape(goal_shape)                       # (1,384,224,6)
        goal_tensor = tf.convert_to_tensor(goal_data, dtype=tf.float32) # (1,384,224,6)

        # Get SE2 rotation vectors for cropping.
        pivot = np.array([p[1], p[0]]) + self.pad_size
        rvecs = self.get_se2(self.num_rotations, pivot)

        # Forward pass through three separate FCNs. All logits will be: (1,384,224,3).
        in_logits, kernel_nocrop_logits, goal_logits = \
                    self.model([in_tensor, in_tensor, goal_tensor])

        # Use features from goal logits and combine with input and kernel.
        goal_x_in_logits     = tf.multiply(goal_logits, in_logits)
        goal_x_kernel_logits = tf.multiply(goal_logits, kernel_nocrop_logits)

        # Crop the kernel_logits about the picking point and get rotations.
        crop = tf.identity(goal_x_kernel_logits)                            # (1,384,224,3)
        crop = tf.repeat(crop, repeats=self.num_rotations, axis=0)          # (24,384,224,3)
        crop = tfa.image.transform(crop, rvecs, interpolation='NEAREST')    # (24,384,224,3)
        kernel = crop[:,
                      p[0]:(p[0] + self.crop_size),
                      p[1]:(p[1] + self.crop_size),
                      :]
        assert kernel.shape == (self.num_rotations, self.crop_size, self.crop_size, self.odim)

        # Cross-convolve `in_x_goal_logits`. Padding kernel: (24,64,64,3) --> (65,65,3,24).
        kernel_paddings = tf.constant([[0, 0], [0, 1], [0, 1], [0, 0]])
        kernel = tf.pad(kernel, kernel_paddings, mode='CONSTANT')
        kernel = tf.transpose(kernel, [1, 2, 3, 0])
        output = tf.nn.convolution(goal_x_in_logits, kernel, data_format="NHWC")
        output = (1 / (self.crop_size**2)) * output

        if apply_softmax:
            output_shape = output.shape
            output = tf.reshape(output, (1, np.prod(output.shape)))
            output = tf.nn.softmax(output)
            output = np.float32(output).reshape(output_shape[1:])

        # Daniel: visualize crops and kernels, for Transporter-Goal figure.
        #self.visualize_images(p, in_img, input_data, crop)
        #self.visualize_transport(p, in_img, input_data, crop, kernel)
        #self.visualize_logits(in_logits,            name='input')
        #self.visualize_logits(goal_logits,          name='goal')
        #self.visualize_logits(kernel_nocrop_logits, name='kernel')
        #self.visualize_logits(goal_x_in_logits,     name='goal_x_in')
        #self.visualize_logits(goal_x_kernel_logits, name='goal_x_kernel')

        return output
Forward pass of our goal-conditioned Transporter.

        Relevant shapes and info:

            in_img and goal_img: (320,160,6)
            p: integer pixels on in_img, e.g., [158, 30]
            self.padding: [[32,32],[32,32],0,0]], with shape (3,2)

        Run input through all three networks, to get output of the same
        shape, except that the last channel is 3 (output_dim). Then, the
        output for one stream has the convolutional kernels for another. Call
        tf.nn.convolution. That's it, and the operation is be differentiable,
        so that gradients apply to all the FCNs.

        I actually think cropping after the query network is easier, because
        otherwise we have to do a forward pass, then call tf.multiply, then
        do another forward pass, which splits up the computation.
        
Forward pass of our goal-conditioned Transporter.
Relevant shapes and info.



Run input through all three networks, to get output of the same
shape, except that the last channel is 3 (output_dim). Then, the
output for one stream has the convolutional kernels for another.

I actually think cropping after the query network is easier, because
otherwise we have to do a forward pass, then call tf.multiply, then
do another forward pass, which splits up the computation.
Forward pass of our goal - conditioned Transporter . Relevant shapes and info . Run input through all three networks to get output of the same shape except that the last channel is 3 ( output_dim ) . Then the output for one stream has the convolutional kernels for another . I actually think cropping after the query network is easier because otherwise we have to do a forward pass then call tf . multiply then do another forward pass which splits up the computation .
def forward(self, in_img, goal_img, p, apply_softmax=True):
        assert in_img.shape == goal_img.shape, f'{in_img.shape}, {goal_img.shape}'
        input_unproc = np.pad(in_img, self.padding, mode='constant')    
        input_data = self.preprocess(input_unproc.copy())               
        input_shape = (1,) + input_data.shape
        input_data = input_data.reshape(input_shape)                    
        in_tensor = tf.convert_to_tensor(input_data, dtype=tf.float32)  
        goal_unproc = np.pad(goal_img, self.padding, mode='constant')   
        goal_data = self.preprocess(goal_unproc.copy())                 
        goal_shape = (1,) + goal_data.shape
        goal_data = goal_data.reshape(goal_shape)                       
        goal_tensor = tf.convert_to_tensor(goal_data, dtype=tf.float32) 
        pivot = np.array([p[1], p[0]]) + self.pad_size
        rvecs = self.get_se2(self.num_rotations, pivot)
        in_logits, kernel_nocrop_logits, goal_logits = \
                    self.model([in_tensor, in_tensor, goal_tensor])
        goal_x_in_logits     = tf.multiply(goal_logits, in_logits)
        goal_x_kernel_logits = tf.multiply(goal_logits, kernel_nocrop_logits)
        crop = tf.identity(goal_x_kernel_logits)                            
        crop = tf.repeat(crop, repeats=self.num_rotations, axis=0)          
        crop = tfa.image.transform(crop, rvecs, interpolation='NEAREST')    
        kernel = crop[:,
                      p[0]:(p[0] + self.crop_size),
                      p[1]:(p[1] + self.crop_size),
                      :]
        assert kernel.shape == (self.num_rotations, self.crop_size, self.crop_size, self.odim)
        kernel_paddings = tf.constant([[0, 0], [0, 1], [0, 1], [0, 0]])
        kernel = tf.pad(kernel, kernel_paddings, mode='CONSTANT')
        kernel = tf.transpose(kernel, [1, 2, 3, 0])
        output = tf.nn.convolution(goal_x_in_logits, kernel, data_format="NHWC")
        output = (1 / (self.crop_size**2)) * output
        if apply_softmax:
            output_shape = output.shape
            output = tf.reshape(output, (1, np.prod(output.shape)))
            output = tf.nn.softmax(output)
            output = np.float32(output).reshape(output_shape[1:])
        return output
def forward ( self , in_img , goal_img , p , apply_softmax = True ) : assert in_img . shape == goal_img . shape , f'{in_img.shape}, {goal_img.shape}' input_unproc = np . pad ( in_img , self . padding , mode = 'constant' ) input_data = self . preprocess ( input_unproc . copy ( ) ) input_shape = ( 1 , ) + input_data . shape input_data = input_data . reshape ( input_shape ) in_tensor = tf . convert_to_tensor ( input_data , dtype = tf . float32 ) goal_unproc = np . pad ( goal_img , self . padding , mode = 'constant' ) goal_data = self . preprocess ( goal_unproc . copy ( ) ) goal_shape = ( 1 , ) + goal_data . shape goal_data = goal_data . reshape ( goal_shape ) goal_tensor = tf . convert_to_tensor ( goal_data , dtype = tf . float32 ) pivot = np . array ( [ p [ 1 ] , p [ 0 ] ] ) + self . pad_size rvecs = self . get_se2 ( self . num_rotations , pivot ) in_logits , kernel_nocrop_logits , goal_logits = self . model ( [ in_tensor , in_tensor , goal_tensor ] ) goal_x_in_logits = tf . multiply ( goal_logits , in_logits ) goal_x_kernel_logits = tf . multiply ( goal_logits , kernel_nocrop_logits ) crop = tf . identity ( goal_x_kernel_logits ) crop = tf . repeat ( crop , repeats = self . num_rotations , axis = 0 ) crop = tfa . image . transform ( crop , rvecs , interpolation = 'NEAREST' ) kernel = crop [ : , p [ 0 ] : ( p [ 0 ] + self . crop_size ) , p [ 1 ] : ( p [ 1 ] + self . crop_size ) , : ] assert kernel . shape == ( self . num_rotations , self . crop_size , self . crop_size , self . odim ) kernel_paddings = tf . constant ( [ [ 0 , 0 ] , [ 0 , 1 ] , [ 0 , 1 ] , [ 0 , 0 ] ] ) kernel = tf . pad ( kernel , kernel_paddings , mode = 'CONSTANT' ) kernel = tf . transpose ( kernel , [ 1 , 2 , 3 , 0 ] ) output = tf . nn . convolution ( goal_x_in_logits , kernel , data_format = "NHWC" ) output = ( 1 / ( self . crop_size ** 2 ) ) * output if apply_softmax : output_shape = output . shape output = tf . reshape ( output , ( 1 , np . prod ( output . shape ) ) ) output = tf . nn . softmax ( output ) output = np . float32 ( output ) . reshape ( output_shape [ 1 : ] ) return output
Forward pass of our goal-conditioned Transporter.
Forward pass of our goal - conditioned Transporter .
"""Forward pass of our goal-conditioned Transporter.

        Relevant shapes and info:

            in_img and goal_img: (320,160,6)
            p: integer pixels on in_img, e.g., [158, 30]
            self.padding: [[32,32],[32,32],0,0]], with shape (3,2)

        Run input through all three networks, to get output of the same
        shape, except that the last channel is 3 (output_dim). Then, the
        output for one stream has the convolutional kernels for another. Call
        tf.nn.convolution. That's it, and the operation is be differentiable,
        so that gradients apply to all the FCNs.

        I actually think cropping after the query network is easier, because
        otherwise we have to do a forward pass, then call tf.multiply, then
        do another forward pass, which splits up the computation.
        """ # input image --> TF tensor # (384,224,6) # (384,224,6) # (1,384,224,6) # (1,384,224,6) # goal image --> TF tensor # (384,224,6) # (384,224,6) # (1,384,224,6) # (1,384,224,6) # Get SE2 rotation vectors for cropping. # Forward pass through three separate FCNs. All logits will be: (1,384,224,3). # Use features from goal logits and combine with input and kernel. # Crop the kernel_logits about the picking point and get rotations. # (1,384,224,3) # (24,384,224,3) # (24,384,224,3) # Cross-convolve `in_x_goal_logits`. Padding kernel: (24,64,64,3) --> (65,65,3,24). # Daniel: visualize crops and kernels, for Transporter-Goal figure. #self.visualize_images(p, in_img, input_data, crop) #self.visualize_transport(p, in_img, input_data, crop, kernel) #self.visualize_logits(in_logits,            name='input') #self.visualize_logits(goal_logits,          name='goal') #self.visualize_logits(kernel_nocrop_logits, name='kernel') #self.visualize_logits(goal_x_in_logits,     name='goal_x_in') #self.visualize_logits(goal_x_kernel_logits, name='goal_x_kernel')
self
in_img
goal_img
p
apply_softmax