class_docstr: RCNN: Region-based Convolutional Neural Network.
  
  Given region proposals (bounding boxes on an image) and a feature map of
  that image, RCNN adjusts the bounding boxes and classifies each region as
  either background or a specific object class.
  
  Steps:
      1. Region of Interest Pooling. Extract features from the feature map
         (based on the proposals) and convert into fixed size tensors
         (applying extrapolation).
      2. Two fully connected layers generate a smaller tensor for each
         region.
      3. A fully conected layer outputs the probability distribution over the
         classes (plus a background class), and another fully connected layer
         outputs the bounding box regressions (one 4-d regression for each of
         the possible classes).
  
  Using the class probability, filter regions classified as background. For
  the remaining regions, use the class probability together with the
  corresponding bounding box regression offsets to generate the final object
  bounding boxes, with classes and probabilities assigned.
class_docstr_tok: ["RCNN",":","Region-based","Convolutional","Neural","Network",".","Given","region","proposals","(","bounding","boxes","on","an","image",")","and","a","feature","map","of","that","image",",","RCNN","adjusts","the","bounding","boxes","and","classifies","each","region","as","either","background","or","a","specific","object","class",".","Steps",":","1",".","Region","of","Interest","Pooling",".","Extract","features","from","the","feature","map","(","based","on","the","proposals",")","and","convert","into","fixed","size","tensors","(","applying","extrapolation",")",".","2",".","Two","fully","connected","layers","generate","a","smaller","tensor","for","each","region",".","3",".","A","fully","conected","layer","outputs","the","probability","distribution","over","the","classes","(","plus","a","background","class",")",",","and","another","fully","connected","layer","outputs","the","bounding","box","regressions","(","one","4-d","regression","for","each","of","the","possible","classes",")",".","Using","the","class","probability",",","filter","regions","classified","as","background",".","For","the","remaining","regions",",","use","the","class","probability","together","with","the","corresponding","bounding","box","regression","offsets","to","generate","the","final","object","bounding","boxes",",","with","classes","and","probabilities","assigned","."]
docstr: Returns cost for RCNN based on:
  
  Args:
      prediction_dict with keys:
          rcnn:
              cls_score: shape (num_proposals, num_classes + 1)
                  Has the class scoring for each the proposals. Classes
                  are 1-indexed with 0 being the background.
  
              cls_prob: shape (num_proposals, num_classes + 1)
                  Application of softmax on cls_score.
  
              bbox_offsets: shape (num_proposals, num_classes * 4)
                  Has the offset for each proposal for each class.
                  We have to compare only the proposals labeled with the
                  offsets for that label.
  
          target:
              cls_target: shape (num_proposals,)
                  Has the correct label for each of the proposals.
                  0 => background
                  1..n => 1-indexed classes
  
              bbox_offsets_target: shape (num_proposals, 4)
                  Has the true offset of each proposal for the true
                  label.
                  In case of not having a true label (non-background)
                  then it's just zeroes.
  
  Returns:
      loss_dict with keys:
          rcnn_cls_loss: The cross-entropy or log-loss of the
              classification tasks between then num_classes + background.
          rcnn_reg_loss: The smooth L1 loss for the bounding box
              regression task to adjust correctly labeled boxes.
docstr_tok: ["Returns","cost","for","RCNN","based","on",":","Args",":","prediction_dict","with","keys",":","rcnn",":","cls_score",":","shape","(","num_proposals",",","num_classes","+","1",")","Has","the","class","scoring","for","each","the","proposals",".","Classes","are","1-indexed","with","0","being","the","background",".","cls_prob",":","shape","(","num_proposals",",","num_classes","+","1",")","Application","of","softmax","on","cls_score",".","bbox_offsets",":","shape","(","num_proposals",",","num_classes","*","4",")","Has","the","offset","for","each","proposal","for","each","class",".","We","have","to","compare","only","the","proposals","labeled","with","the","offsets","for","that","label",".","target",":","cls_target",":","shape","(","num_proposals",",",")","Has","the","correct","label","for","each","of","the","proposals",".","0","=",">","background","1","..","n","=",">","1-indexed","classes","bbox_offsets_target",":","shape","(","num_proposals",",","4",")","Has","the","true","offset","of","each","proposal","for","the","true","label",".","In","case","of","not","having","a","true","label","(","non-background",")","then","it","'s","just","zeroes",".","Returns",":","loss_dict","with","keys",":","rcnn_cls_loss",":","The","cross-entropy","or","log-loss","of","the","classification","tasks","between","then","num_classes","+","background",".","rcnn_reg_loss",":","The","smooth","L1","loss","for","the","bounding","box","regression","task","to","adjust","correctly","labeled","boxes","."]
code: def loss(self, prediction_dict):
      """
      Returns cost for RCNN based on:
  
      Args:
          prediction_dict with keys:
              rcnn:
                  cls_score: shape (num_proposals, num_classes + 1)
                      Has the class scoring for each the proposals. Classes
                      are 1-indexed with 0 being the background.
  
                  cls_prob: shape (num_proposals, num_classes + 1)
                      Application of softmax on cls_score.
  
                  bbox_offsets: shape (num_proposals, num_classes * 4)
                      Has the offset for each proposal for each class.
                      We have to compare only the proposals labeled with the
                      offsets for that label.
  
              target:
                  cls_target: shape (num_proposals,)
                      Has the correct label for each of the proposals.
                      0 => background
                      1..n => 1-indexed classes
  
                  bbox_offsets_target: shape (num_proposals, 4)
                      Has the true offset of each proposal for the true
                      label.
                      In case of not having a true label (non-background)
                      then it's just zeroes.
  
      Returns:
          loss_dict with keys:
              rcnn_cls_loss: The cross-entropy or log-loss of the
                  classification tasks between then num_classes + background.
              rcnn_reg_loss: The smooth L1 loss for the bounding box
                  regression task to adjust correctly labeled boxes.
  
      """
      with tf.name_scope("RCNNLoss"):
          cls_score = prediction_dict["rcnn"]["cls_score"]
          # cls_prob = prediction_dict['rcnn']['cls_prob']
          # Cast target explicitly as int32.
          cls_target = tf.cast(
              prediction_dict["target"]["cls"], tf.int32
          )
  
          # First we need to calculate the log loss betweetn cls_prob and
          # cls_target
  
          # We only care for the targets that are >= 0
          not_ignored = tf.reshape(
              tf.greater_equal(cls_target, 0), [-1], name="not_ignored"
          )
          # We apply boolean mask to score, prob and target.
          cls_score_labeled = tf.boolean_mask(
              cls_score, not_ignored, name="cls_score_labeled"
          )
          # cls_prob_labeled = tf.boolean_mask(
          #    cls_prob, not_ignored, name='cls_prob_labeled')
          cls_target_labeled = tf.boolean_mask(
              cls_target, not_ignored, name="cls_target_labeled"
          )
  
          tf.summary.scalar(
              "batch_size", tf.shape(cls_score_labeled)[0], ["rcnn"]
          )
  
          # Transform to one-hot vector
          cls_target_one_hot = tf.one_hot(
              cls_target_labeled,
              depth=self._num_classes + 1,
              name="cls_target_one_hot",
          )
  
          # We get cross entropy loss of each proposal.
          cross_entropy_per_proposal = (
              tf.nn.softmax_cross_entropy_with_logits_v2(
                  labels=tf.stop_gradient(cls_target_one_hot),
                  logits=cls_score_labeled,
              )
          )
  
          if self._debug:
              prediction_dict["_debug"]["losses"] = {}
              # Save the cross entropy per proposal to be able to
              # visualize proposals with high and low error.
              prediction_dict["_debug"]["losses"][
                  "cross_entropy_per_proposal"
              ] = cross_entropy_per_proposal
  
          # Second we need to calculate the smooth l1 loss between
          # `bbox_offsets` and `bbox_offsets_target`.
          bbox_offsets = prediction_dict["rcnn"]["bbox_offsets"]
          bbox_offsets_target = prediction_dict["target"][
              "bbox_offsets"
          ]
  
          # We only want the non-background labels bounding boxes.
          not_ignored = tf.reshape(tf.greater(cls_target, 0), [-1])
          bbox_offsets_labeled = tf.boolean_mask(
              bbox_offsets, not_ignored, name="bbox_offsets_labeled"
          )
          bbox_offsets_target_labeled = tf.boolean_mask(
              bbox_offsets_target,
              not_ignored,
              name="bbox_offsets_target_labeled",
          )
  
          cls_target_labeled = tf.boolean_mask(
              cls_target, not_ignored, name="cls_target_labeled"
          )
          # `cls_target_labeled` is based on `cls_target` which has
          # `num_classes` + 1 classes.
          # for making `one_hot` with depth `num_classes` to work we need
          # to lower them to make them 0-index.
          cls_target_labeled = cls_target_labeled - 1
  
          cls_target_one_hot = tf.one_hot(
              cls_target_labeled,
              depth=self._num_classes,
              name="cls_target_one_hot",
          )
  
          # cls_target now is (num_labeled, num_classes)
          bbox_flatten = tf.reshape(
              bbox_offsets_labeled, [-1, 4], name="bbox_flatten"
          )
  
          # We use the flatten cls_target_one_hot as boolean mask for the
          # bboxes.
          cls_flatten = tf.cast(
              tf.reshape(cls_target_one_hot, [-1]),
              tf.bool,
              "cls_flatten_as_bool",
          )
  
          bbox_offset_cleaned = tf.boolean_mask(
              bbox_flatten, cls_flatten, "bbox_offset_cleaned"
          )
  
          # Calculate the smooth l1 loss between the "cleaned" bboxes
          # offsets (that means, the useful results) and the labeled
          # targets.
          reg_loss_per_proposal = smooth_l1_loss(
              bbox_offset_cleaned,
              bbox_offsets_target_labeled,
              sigma=self._l1_sigma,
          )
  
          tf.summary.scalar(
              "rcnn_foreground_samples",
              tf.shape(bbox_offset_cleaned)[0],
              ["rcnn"],
          )
  
          if self._debug:
              # Also save reg loss per proposals to be able to visualize
              # good and bad proposals in debug mode.
              prediction_dict["_debug"]["losses"][
                  "reg_loss_per_proposal"
              ] = reg_loss_per_proposal
  
          return {
              "rcnn_cls_loss": tf.reduce_mean(
                  cross_entropy_per_proposal
              ),
              "rcnn_reg_loss": tf.reduce_mean(reg_loss_per_proposal),
          }
  
file_path: luminoth/luminoth/models/fasterrcnn/rcnn.py
imports: import sonnet
  import tensorflow
  from luminoth.models.fasterrcnn.rcnn_proposal import RCNNProposal
  from luminoth.models.fasterrcnn.rcnn_target import RCNNTarget
  from luminoth.models.fasterrcnn.roi_pool import ROIPoolingLayer
  from luminoth.utils.losses import smooth_l1_loss
  from luminoth.utils.vars import get_initializer, layer_summaries, variable_summaries, get_activation_function
label_desc: Use image node_id 4 for calling the RCNN obj's underlying member method code with example usage: obj.loss(prediction_dict) and returns: dict