ravens/tasks/defs_bags.py
def is_item_in_bag(self):
        """Used to determine reward. We want item to be inside the bag hull.

        Actually, first detect bead visibility. If we can't see any then we should
        just return 0 because that means we've messed up somewhere.

        Returns an info dict, which has 'reward' keyword.
        """
        result = {'exit_early': False,
                  'frac_in_target_bag': 0.0,
                  'frac_in_distract_bag': 0.0}
        colormap, heightmap, object_mask = self.get_object_masks(self.env)

        # Detect visible beads for the FIRST bag! The second bag is irrelevant.
        visible_beads = []
        for bead in self.cable_bead_target_bag_IDs:
            if bead in object_mask:
                visible_beads.append(bead)

        # Check object_mask for all IDs that correspond to the cable ring.
        cable_IDs = np.array(self.cable_bead_target_bag_IDs)
        bead_mask = np.isin(object_mask, test_elements=cable_IDs)

        # Threshold image to get 0s and 255s (255s=bead pixels) and find its contours.
        bead_mask = np.uint8(bead_mask * 255)
        contours, _ = cv2.findContours(bead_mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)

        # If only a few beads are visible (or no contours detected) exit early.
        # NOTE: KEEP THE KEYS CONSISTENT WITH THIS AND THE RESULT!
        frac_visible = len(visible_beads) / len(self.cable_bead_target_bag_IDs)
        result['beads_visible'] = len(visible_beads)
        if len(contours) == 0 or frac_visible <= BEAD_THRESH:
            result['exit_early'] = True
            return result

        # Combine contours via concatenation (shape=(N,1,2)) and get the convex hull.
        allc = np.concatenate([c for c in contours])
        contours_list = [allc]
        hull_list = [cv2.convexHull(c) for c in contours_list]

        # Make an RGB image, then draw the filled-in area of all items in `hull_list`.
        hull = np.zeros((bead_mask.shape[0], bead_mask.shape[1], 3), dtype=np.uint8)
        cv2.drawContours(hull, hull_list, -1, (255,255,255), thickness=-1)
        hull = cv2.cvtColor(hull, cv2.COLOR_BGR2GRAY)

        # (now comes the new stuff for the reward vs the task determination stage)
        # next, find the pixels of the target item, and determine fraction that coincide w/the target hull.
        pixels_target_item    = np.uint8(object_mask == self.single_block_ID)
        target_and_hull       = pixels_target_item & hull  # intersection of target item's and hull's pixels
        target_and_hull_count = np.count_nonzero(target_and_hull)  # number of pixels for this intersection
        target_pixel_count    = np.count_nonzero(pixels_target_item)  # number of pixels for target item
        frac_in_target_bag    = target_and_hull_count / (float(target_pixel_count) + 0.01)

        # ----------------------------------------------------------------------------- #
        # DISTRACTORS: now check hull of the OTHER bag ... ONLY for debugging purposes. #
        # ----------------------------------------------------------------------------- #
        visible_beads = []
        for bead in self.cable_bead_IDs[self.beads_per_bag:]:
            if bead in object_mask:
                visible_beads.append(bead)

        # Check object_mask for all IDs that correspond to the cable ring.
        cable_IDs = np.array( self.cable_bead_IDs[self.beads_per_bag:] )
        bead_mask = np.isin(object_mask, test_elements=cable_IDs)

        # Threshold image to get 0s and 255s (255s=bead pixels) and find its contours.
        bead_mask = np.uint8(bead_mask * 255)
        contours, _ = cv2.findContours(bead_mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)

        # If only a few beads are visible (or no contours detected) then this fraction is 0.
        frac_visible = len(visible_beads) / self.beads_per_bag
        if len(contours) == 0 or frac_visible <= BEAD_THRESH:
            frac_in_distract_bag = 0.0
        else:
            # Combine contours via concatenation (shape=(N,1,2)) and get the convex hull.
            allc = np.concatenate([c for c in contours])
            contours_list = [allc]
            hull_list = [cv2.convexHull(c) for c in contours_list]

            # Make an RGB image, then draw the filled-in area of all items in `hull_list`.
            hull = np.zeros((bead_mask.shape[0], bead_mask.shape[1], 3), dtype=np.uint8)
            cv2.drawContours(hull, hull_list, -1, (255,255,255), thickness=-1)
            hull = cv2.cvtColor(hull, cv2.COLOR_BGR2GRAY)

            # We use pixels_target_item from earlier, just a different hull now.
            target_and_hull       = pixels_target_item & hull  # intersection of target item's and hull's pixels
            target_and_hull_count = np.count_nonzero(target_and_hull)  # number of pixels for this intersection
            target_pixel_count    = np.count_nonzero(pixels_target_item)  # number of pixels for target item
            frac_in_distract_bag  = target_and_hull_count / (float(target_pixel_count) + 0.01)

        # TODO(daniel) ensure that this matches the keys from exiting early!
        #result['frac_distract_in_hull'] = frac_distract_in_hull
        result['frac_in_target_bag']   = frac_in_target_bag
        result['frac_in_distract_bag'] = frac_in_distract_bag
        return result

        ##place_rgb = cv2.cvtColor(hull.copy(), cv2.COLOR_GRAY2BGR)
        ##place_rgb[pixels_target_item > 0] = 127  # gray for pixel targets
        ##nb = len([x for x in os.listdir('tmp/') if 'color' in x and '.png' in x])
        ##cv2.imwrite(f'tmp/img_{nb}_colormap.png', cv2.cvtColor(colormap, cv2.COLOR_RGB2BGR).astype(np.uint8))
        ##cv2.imwrite(f'tmp/img_{nb}_bead_mask.png', bead_mask)
        ##cv2.imwrite(f'tmp/img_{nb}_place_rgb.png', cv2.cvtColor(place_rgb, cv2.COLOR_RGB2BGR))
Used to determine reward. We want item to be inside the bag hull.

        Actually, first detect bead visibility. If we can't see any then we should
        just return 0 because that means we've messed up somewhere.

        Returns an info dict, which has 'reward' keyword.
        
Used to determine reward. We want item to be inside the bag hull.
Actually, first detect bead visibility. If we can't see any then we should
just return 0 because that means we've messed up somewhere.

Returns an info dict, which has 'reward' keyword.
Used to determine reward . We want item to be inside the bag hull . Actually first detect bead visibility . If we can ' t see any then we should just return 0 because that means we ' ve messed up somewhere . Returns an info dict which has ' reward ' keyword .
def is_item_in_bag(self):
        result = {'exit_early': False,
                  'frac_in_target_bag': 0.0,
                  'frac_in_distract_bag': 0.0}
        colormap, heightmap, object_mask = self.get_object_masks(self.env)
        visible_beads = []
        for bead in self.cable_bead_target_bag_IDs:
            if bead in object_mask:
                visible_beads.append(bead)
        cable_IDs = np.array(self.cable_bead_target_bag_IDs)
        bead_mask = np.isin(object_mask, test_elements=cable_IDs)
        bead_mask = np.uint8(bead_mask * 255)
        contours, _ = cv2.findContours(bead_mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
        frac_visible = len(visible_beads) / len(self.cable_bead_target_bag_IDs)
        result['beads_visible'] = len(visible_beads)
        if len(contours) == 0 or frac_visible <= BEAD_THRESH:
            result['exit_early'] = True
            return result
        allc = np.concatenate([c for c in contours])
        contours_list = [allc]
        hull_list = [cv2.convexHull(c) for c in contours_list]
        hull = np.zeros((bead_mask.shape[0], bead_mask.shape[1], 3), dtype=np.uint8)
        cv2.drawContours(hull, hull_list, -1, (255,255,255), thickness=-1)
        hull = cv2.cvtColor(hull, cv2.COLOR_BGR2GRAY)
        pixels_target_item    = np.uint8(object_mask == self.single_block_ID)
        target_and_hull       = pixels_target_item & hull  
        target_and_hull_count = np.count_nonzero(target_and_hull)  
        target_pixel_count    = np.count_nonzero(pixels_target_item)  
        frac_in_target_bag    = target_and_hull_count / (float(target_pixel_count) + 0.01)
        visible_beads = []
        for bead in self.cable_bead_IDs[self.beads_per_bag:]:
            if bead in object_mask:
                visible_beads.append(bead)
        cable_IDs = np.array( self.cable_bead_IDs[self.beads_per_bag:] )
        bead_mask = np.isin(object_mask, test_elements=cable_IDs)
        bead_mask = np.uint8(bead_mask * 255)
        contours, _ = cv2.findContours(bead_mask, cv2.RETR_TREE, cv2.CHAIN_APPROX_SIMPLE)
        frac_visible = len(visible_beads) / self.beads_per_bag
        if len(contours) == 0 or frac_visible <= BEAD_THRESH:
            frac_in_distract_bag = 0.0
        else:
            allc = np.concatenate([c for c in contours])
            contours_list = [allc]
            hull_list = [cv2.convexHull(c) for c in contours_list]
            hull = np.zeros((bead_mask.shape[0], bead_mask.shape[1], 3), dtype=np.uint8)
            cv2.drawContours(hull, hull_list, -1, (255,255,255), thickness=-1)
            hull = cv2.cvtColor(hull, cv2.COLOR_BGR2GRAY)
            target_and_hull       = pixels_target_item & hull  
            target_and_hull_count = np.count_nonzero(target_and_hull)  
            target_pixel_count    = np.count_nonzero(pixels_target_item)  
            frac_in_distract_bag  = target_and_hull_count / (float(target_pixel_count) + 0.01)
        result['frac_in_target_bag']   = frac_in_target_bag
        result['frac_in_distract_bag'] = frac_in_distract_bag
        return result
def is_item_in_bag ( self ) : result = { 'exit_early' : False , 'frac_in_target_bag' : 0.0 , 'frac_in_distract_bag' : 0.0 } colormap , heightmap , object_mask = self . get_object_masks ( self . env ) visible_beads = [ ] for bead in self . cable_bead_target_bag_IDs : if bead in object_mask : visible_beads . append ( bead ) cable_IDs = np . array ( self . cable_bead_target_bag_IDs ) bead_mask = np . isin ( object_mask , test_elements = cable_IDs ) bead_mask = np . uint8 ( bead_mask * 255 ) contours , _ = cv2 . findContours ( bead_mask , cv2 . RETR_TREE , cv2 . CHAIN_APPROX_SIMPLE ) frac_visible = len ( visible_beads ) / len ( self . cable_bead_target_bag_IDs ) result [ 'beads_visible' ] = len ( visible_beads ) if len ( contours ) == 0 or frac_visible <= BEAD_THRESH : result [ 'exit_early' ] = True return result allc = np . concatenate ( [ c for c in contours ] ) contours_list = [ allc ] hull_list = [ cv2 . convexHull ( c ) for c in contours_list ] hull = np . zeros ( ( bead_mask . shape [ 0 ] , bead_mask . shape [ 1 ] , 3 ) , dtype = np . uint8 ) cv2 . drawContours ( hull , hull_list , - 1 , ( 255 , 255 , 255 ) , thickness = - 1 ) hull = cv2 . cvtColor ( hull , cv2 . COLOR_BGR2GRAY ) pixels_target_item = np . uint8 ( object_mask == self . single_block_ID ) target_and_hull = pixels_target_item & hull target_and_hull_count = np . count_nonzero ( target_and_hull ) target_pixel_count = np . count_nonzero ( pixels_target_item ) frac_in_target_bag = target_and_hull_count / ( float ( target_pixel_count ) + 0.01 ) visible_beads = [ ] for bead in self . cable_bead_IDs [ self . beads_per_bag : ] : if bead in object_mask : visible_beads . append ( bead ) cable_IDs = np . array ( self . cable_bead_IDs [ self . beads_per_bag : ] ) bead_mask = np . isin ( object_mask , test_elements = cable_IDs ) bead_mask = np . uint8 ( bead_mask * 255 ) contours , _ = cv2 . findContours ( bead_mask , cv2 . RETR_TREE , cv2 . CHAIN_APPROX_SIMPLE ) frac_visible = len ( visible_beads ) / self . beads_per_bag if len ( contours ) == 0 or frac_visible <= BEAD_THRESH : frac_in_distract_bag = 0.0 else : allc = np . concatenate ( [ c for c in contours ] ) contours_list = [ allc ] hull_list = [ cv2 . convexHull ( c ) for c in contours_list ] hull = np . zeros ( ( bead_mask . shape [ 0 ] , bead_mask . shape [ 1 ] , 3 ) , dtype = np . uint8 ) cv2 . drawContours ( hull , hull_list , - 1 , ( 255 , 255 , 255 ) , thickness = - 1 ) hull = cv2 . cvtColor ( hull , cv2 . COLOR_BGR2GRAY ) target_and_hull = pixels_target_item & hull target_and_hull_count = np . count_nonzero ( target_and_hull ) target_pixel_count = np . count_nonzero ( pixels_target_item ) frac_in_distract_bag = target_and_hull_count / ( float ( target_pixel_count ) + 0.01 ) result [ 'frac_in_target_bag' ] = frac_in_target_bag result [ 'frac_in_distract_bag' ] = frac_in_distract_bag return result
Used to determine reward.
Used to determine reward .
"""Used to determine reward. We want item to be inside the bag hull.

        Actually, first detect bead visibility. If we can't see any then we should
        just return 0 because that means we've messed up somewhere.

        Returns an info dict, which has 'reward' keyword.
        """ # Detect visible beads for the FIRST bag! The second bag is irrelevant. # Check object_mask for all IDs that correspond to the cable ring. # Threshold image to get 0s and 255s (255s=bead pixels) and find its contours. # If only a few beads are visible (or no contours detected) exit early. # NOTE: KEEP THE KEYS CONSISTENT WITH THIS AND THE RESULT! # Combine contours via concatenation (shape=(N,1,2)) and get the convex hull. # Make an RGB image, then draw the filled-in area of all items in `hull_list`. # (now comes the new stuff for the reward vs the task determination stage) # next, find the pixels of the target item, and determine fraction that coincide w/the target hull. # intersection of target item's and hull's pixels # number of pixels for this intersection # number of pixels for target item # ----------------------------------------------------------------------------- # # DISTRACTORS: now check hull of the OTHER bag ... ONLY for debugging purposes. # # ----------------------------------------------------------------------------- # # Check object_mask for all IDs that correspond to the cable ring. # Threshold image to get 0s and 255s (255s=bead pixels) and find its contours. # If only a few beads are visible (or no contours detected) then this fraction is 0. # Combine contours via concatenation (shape=(N,1,2)) and get the convex hull. # Make an RGB image, then draw the filled-in area of all items in `hull_list`. # We use pixels_target_item from earlier, just a different hull now. # intersection of target item's and hull's pixels # number of pixels for this intersection # number of pixels for target item # TODO(daniel) ensure that this matches the keys from exiting early! #result['frac_distract_in_hull'] = frac_distract_in_hull ##place_rgb = cv2.cvtColor(hull.copy(), cv2.COLOR_GRAY2BGR) ##place_rgb[pixels_target_item > 0] = 127  # gray for pixel targets ##nb = len([x for x in os.listdir('tmp/') if 'color' in x and '.png' in x]) ##cv2.imwrite(f'tmp/img_{nb}_colormap.png', cv2.cvtColor(colormap, cv2.COLOR_RGB2BGR).astype(np.uint8)) ##cv2.imwrite(f'tmp/img_{nb}_bead_mask.png', bead_mask) ##cv2.imwrite(f'tmp/img_{nb}_place_rgb.png', cv2.cvtColor(place_rgb, cv2.COLOR_RGB2BGR))
self