text: """
  The script is slightly modified to allow training on fruits.
  The original file is written in the ballon.py, which is written by Waleed Abdulla.
  Hence, his the work is honoured and credit must be given to the original author
  
  honour the original author:
  Copyright (c) 2018 Matterport, Inc.
  Licensed under the MIT License (see LICENSE for details)
  Written by Waleed Abdulla
  
  ------------------------------------------------------------
  
  Usage: import the module (see Jupyter notebooks for examples), or run from
         the command line as such:
  
      # Train a new model starting from pre-trained COCO weights
      python3 balloon.py train --dataset=/path/to/balloon/dataset --weights=coco
  
      # Resume training a model that you had trained earlier
      python3 balloon.py train --dataset=/path/to/balloon/dataset --weights=last
  
      # Train a new model starting from ImageNet weights
      python3 balloon.py train --dataset=/path/to/balloon/dataset --weights=imagenet
  
      # Apply color splash to an image
      python3 balloon.py splash --weights=/path/to/weights/file.h5 --image=<URL or path to file>
  
      # Apply color splash to video using the last weights you trained
      python3 balloon.py splash --weights=last --video=<URL or path to file>
  """
  
  import os
  import sys
  import json
  import datetime
  import numpy as np
  import skimage.draw
  from imgaug import augmenters as iaa
  
  # Root directory of the project
  ROOT_DIR = os.path.abspath("../../")
  
  # Import Mask RCNN
  sys.path.append(ROOT_DIR)  # To find local version of the library
  from mrcnn.config import Config
  from mrcnn import model as modellib, utils
  
  # Path to trained weights file
  COCO_WEIGHTS_PATH = os.path.join(ROOT_DIR, "mask_rcnn_coco.h5")
  
  # Directory to save logs and model checkpoints, if not provided
  # through the command line argument --logs
  DEFAULT_LOGS_DIR = os.path.join(ROOT_DIR, "logs")
  
  ############################################################
  #  Configurations
  ############################################################
  
  
  class FruitConfig(Config):
      """Configuration for training on the toy  dataset.
      Derives from the base Config class and overrides some values.
      """
      # Give the configuration a recognizable name
      NAME = "fruit"
  
      # NUMBER OF GPUs to use. When using only a CPU, this needs to be set to 1.
      GPU_COUNT = 1
  
      # We use a GPU with 12GB memory, which can fit two images.
      # Adjust down if you use a smaller GPU.
      IMAGES_PER_GPU = 1
  
      # Backbone network architecture
      # Supported values are: resnet50, resnet101
      BACKBONE = "resnet101" # used for head in first version of dataset
      # BACKBONE = "resnet50"
  
      # Number of classes (including background)
      NUM_CLASSES = 1 + 2  # Background + banana + pear
  
      # Input image resizing
      # Generally, use the "square" resizing mode for training and predicting
      # and it should work well in most cases. In this mode, images are scaled
      # up such that the small side is = IMAGE_MIN_DIM, but ensuring that the
      # scaling doesn't make the long side > IMAGE_MAX_DIM. Then the image is
      # padded with zeros to make it a square so multiple images can be put
      # in one batch.
      # Available resizing modes:
      # none:   No resizing or padding. Return the image unchanged.
      # square: Resize and pad with zeros to get a square image
      #         of size [max_dim, max_dim].
      # pad64:  Pads width and height with zeros to make them multiples of 64.
      #         If IMAGE_MIN_DIM or IMAGE_MIN_SCALE are not None, then it scales
      #         up before padding. IMAGE_MAX_DIM is ignored in this mode.
      #         The multiple of 64 is needed to ensure smooth scaling of feature
      #         maps up and down the 6 levels of the FPN pyramid (2**6=64).
      # crop:   Picks random crops from the image. First, scales the image based
      #         on IMAGE_MIN_DIM and IMAGE_MIN_SCALE, then picks a random crop of
      #         size IMAGE_MIN_DIM x IMAGE_MIN_DIM. Can be used in training only.
      #         IMAGE_MAX_DIM is not used in this mode.
      IMAGE_RESIZE_MODE = "square"
      IMAGE_MIN_DIM = 800
      IMAGE_MAX_DIM = 1024 # 1600 for heads, 1280for 4+, 1024 works for all
  
      # Image mean (RGB)
      MEAN_PIXEL = np.array([123.7, 116.8, 103.9])
  
      # Number of ROIs per image to feed to classifier/mask heads
      # The Mask RCNN paper uses 512 but often the RPN doesn't generate
      # enough positive proposals to fill this and keep a positive:negative
      # ratio of 1:3. You can increase the number of proposals by adjusting
      # the RPN NMS threshold.
      TRAIN_ROIS_PER_IMAGE = 200 # 512 for heads. 200 for 4+ & all
  
      # Non-max suppression threshold to filter RPN proposals.
      # You can increase this during training to generate more proposals.
      RPN_NMS_THRESHOLD = 0.7 # default
  
      # ROIs kept after non-maximum suppression (training and inference)
      POST_NMS_ROIS_TRAINING = 2000  # 2000 for heads,4+ & all
  
      # How many anchors per image to use for RPN training
      RPN_TRAIN_ANCHORS_PER_IMAGE = 256 # 320 for heads,256 for 3+ & all
  
      # Minimum probability value to accept a detected instance
      # ROIs below this threshold are skipped
      DETECTION_MIN_CONFIDENCE = 0.7
  
      # Maximum number of ground truth instances to use in one image
      # don't think an identify-able image can hold >200 fruit instances
      MAX_GT_INSTANCES = 100 # was 200 for head, 150 for 4+, 100 for all
  
      # Number of training steps per epoch
      STEPS_PER_EPOCH = 300
      VALIDATION_STEPS = 30
  
      # Learning rate and momentum
      # The Mask RCNN paper uses lr=0.02, but on TensorFlow it causes
      # weights to explode. Likely due to differences in optimizer
      # implementation.
      LEARNING_RATE = 0.001
      LEARNING_MOMENTUM = 0.9
  
      # Weight decay regularization
      # affect L2 strength and is a good way of preventing overfitting.
      # try 0.01, 0.005 and 0.001 first, then try more precisely.
  
      WEIGHT_DECAY = 0.0001
  
      # Max number of final detections
      DETECTION_MAX_INSTANCES = 200
  
  
  ############################################################
  #  Dataset
  ############################################################
  
  class FruitDataset(utils.Dataset):
  
      def load_fruit(self, dataset_dir, subset):
          """Load a subset of the Fruit dataset.
          dataset_dir: Root directory of the dataset.
          subset: Subset to load: train or val
          """
          # Add classes. We have only one class to add.
          self.add_class("fruit", 1, "banana")
          self.add_class("fruit", 2, "pear")
  
          # Train or validation dataset?
          assert subset in ["train", "val"]
          dataset_dir = os.path.join(dataset_dir, subset)
  
          # Load annotations
          # VGG Image Annotator (up to version 1.6) saves each image in the form:
          # { 'filename': '28503151_5b5b7ec140_b.jpg',
          #   'regions': {
          #       '0': {
          #           'region_attributes': {},
          #           'shape_attributes': {
          #               'all_points_x': [...],
          #               'all_points_y': [...],
          #               'name': 'polygon'}},
          #       ... more regions ...
          #   },
          #   'size': 100202
          # }
          # We mostly care about the x and y coordinates of each region
          # Note: In VIA 2.0, regions was changed from a dict to a list.
          annotations = json.load(open(os.path.join(dataset_dir, "via_region_data.json")))
          annotations = list(annotations.values())  # don't need the dict keys
  
          # The VIA tool saves images in the JSON even if they don't have any
          # annotations. Skip unannotated images.
          annotations = [a for a in annotations if a['regions']]
          print("the length of annotations is:-----------",len(annotations),"------------------")
  
          # Add images
          for a in annotations:
              # Get the x, y coordinaets of points of the polygons that make up
              # the outline of each object instance. These are stores in the
              # shape_attributes (see json format above)
              # The if condition is needed to support VIA versions 1.x and 2.x.
              if type(a['regions']) is dict:
                  # print("--------------------checked VIA is Dict---------------")
                  polygons = [r['shape_attributes'] for r in a['regions'].values()]
                  name = [r['region_attributes']['name'] for r in a['regions'].values()]
                  name_dict = {"banana": 1, "pear": 2}
                  name_id = [name_dict[a] for a in name]
                  # print("for this image, classified as: -----------------", name_id, "---------------------")
              else:
                  # print("--------------------checked VIA is list---------------")
                  polygons = [r['shape_attributes'] for r in a['regions']]
                  name = [r['region_attributes']['name'] for r in a['regions']]
                  name_dict = {"banana": 1, "pear": 2}
                  name_id = [name_dict[a] for a in name]
                  # print("for this image, classified as: -----------------", name_id, "---------------------")
  
              # load_mask() needs the image size to convert polygons to masks.
              # Unfortunately, VIA doesn't include it in JSON, so we must read
              # the image. This is only managable since the dataset is tiny.
              image_path = os.path.join(dataset_dir, a['filename'])
              image = skimage.io.imread(image_path)
              height, width = image.shape[:2]
  
              self.add_image(
                  "fruit",
                  image_id=a['filename'],  # use file name as a unique image id
                  path=image_path,
                  class_id=name_id,
                  width=width, height=height,
                  polygons=polygons)
          print("!!!!!!load image into annotation finished!!!!!")
  
      def load_mask(self, image_id):
          """Generate instance masks for an image.
         Returns:
          masks: A bool array of shape [height, width, instance count] with
              one mask per instance.
          class_ids: a 1D array of class IDs of the instance masks.
          """
          # If not a Fruit dataset image, delegate to parent class.
          image_info = self.image_info[image_id]
          if image_info["source"] != "fruit":
              return super(self.__class__, self).load_mask(image_id)
  
          name_id = image_info["class_id"]
          # print("loading mask, id is:--------", name_id)
          # Convert polygons to a bitmap mask of shape
          # [height, width, instance_count]
          info = self.image_info[image_id]
          mask = np.zeros([info["height"], info["width"], len(info["polygons"])],
                          dtype=np.uint8)
          class_ids = np.array(name_id, dtype=np.int32)
          print("call in load_mask, the class id is:------------", class_ids)
          for i, p in enumerate(info["polygons"]):
              # Get indexes of pixels inside the polygon and set them to 1
              rr, cc = skimage.draw.polygon(p['all_points_y'], p['all_points_x'])
              mask[rr, cc, i] = 1
  
          # Return mask, and array of class IDs of each instance. Since we have
          # one class ID only, we return an array of 1s
          # return mask.astype(np.bool), np.ones([mask.shape[-1]], dtype=np.int32)
          return (mask.astype(np.bool), class_ids)
  
      def image_reference(self, image_id):
          """Return the path of the image."""
          info = self.image_info[image_id]
          if info["source"] == "fruit":
              return info["path"]
          else:
              super(self.__class__, self).image_reference(image_id)
  
  
  def train(model):
      """Train the model."""
      # Training dataset.
      dataset_train = FruitDataset()
      dataset_train.load_fruit(args.dataset, "train")
      dataset_train.prepare()
  
      # Validation dataset
      dataset_val = FruitDataset()
      dataset_val.load_fruit(args.dataset, "val")
      dataset_val.prepare()
  
      # default augmentation
      # augmentation = iaa.Sometimes(0.5, [
      #     iaa.Fliplr(0.5),
      #     iaa.GaussianBlur(sigma=(0.0, 5.0))
      # ])
  
      # key to handle overfitting on a small training dataset is augmentation
      augmentation = iaa.SomeOf((0, 2), [
          iaa.Fliplr(0.5),
          iaa.Flipud(0.5),
          iaa.OneOf([iaa.Affine(rotate=90),
                     iaa.Affine(rotate=180),
                     iaa.Affine(rotate=270)]),
          # Make some images brighter and some darker.
          # In 20% of all cases, we sample the multiplier once per channel,
          # which can end up changing the color of the images.
          iaa.Multiply((0.8, 1.2)),
          # Small gaussian blur with random sigma between 0 and 0.25.
          # But we only blur about 50% of all images.
          iaa.GaussianBlur(sigma=(0.0, 0.25)),
          # Strengthen or weaken the contrast in each image.
          iaa.ContrastNormalization((0.75, 1.5))
      ])
  
  
      # *** This training schedule is an example. Update to your needs ***
      # Since we're using a very small dataset, and starting from
      # COCO trained weights, we don't need to train too long. Also,
      # no need to train all layers, just the heads should do it.
  
      # MULTIPLE STAGE to help converge easier, since we have limited dataset
  
      # # we have limited training data, so train the classifier only might be a gud idea
      # # this ensure we keep the good coco weights untouched
      # # Training - Stage 1
      # print("Training network heads")
      # model.train(dataset_train, dataset_val,
      #             learning_rate=config.LEARNING_RATE,
      #             epochs=35,
      #             augmentation=augmentation,
      #             layers='heads')
  
      # # Training - Stage 2
      # # Finetune layers from ResNet stage 4 and up
      # print("Fine tune Resnet stage 4 and up")
      # model.train(dataset_train, dataset_val,
      #             learning_rate=config.LEARNING_RATE/10,
      #             epochs=55,
      #             augmentation=augmentation,
      #             layers='4+')
  
      # Training - Stage 3
      # Fine tune all layers
      print("Training all layers")
      model.train(dataset_train, dataset_val,
                  learning_rate=config.LEARNING_RATE/100,
                  epochs=65,
                  augmentation=augmentation,
                  layers='all')
  
  
  def color_splash(image, mask):
      """Apply color splash effect.
      image: RGB image [height, width, 3]
      mask: instance segmentation mask [height, width, instance count]
  
      Returns result image.
      """
      # Make a grayscale copy of the image. The grayscale copy still
      # has 3 RGB channels, though.
      gray = skimage.color.gray2rgb(skimage.color.rgb2gray(image)) * 255
      # Copy color pixels from the original color image where mask is set
      if mask.shape[-1] > 0:
          # We're treating all instances as one, so collapse the mask into one layer
          mask = (np.sum(mask, -1, keepdims=True) >= 1)
          splash = np.where(mask, image, gray).astype(np.uint8)
      else:
          splash = gray.astype(np.uint8)
      return splash
  
  
  def detect_and_color_splash(model, image_path=None, video_path=None):
      assert image_path or video_path
  
      # Image or video?
      if image_path:
          # Run model detection and generate the color splash effect
          print("Running on {}".format(args.image))
          # Read image
          image = skimage.io.imread(args.image)
          # Detect objects
          r = model.detect([image], verbose=1)[0]
          # Color splash
          splash = color_splash(image, r['masks'])
          # Save output
          file_name = "splash_{:%Y%m%dT%H%M%S}.png".format(datetime.datetime.now())
          skimage.io.imsave(file_name, splash)
      elif video_path:
          import cv2
          # Video capture
          vcapture = cv2.VideoCapture(video_path)
          width = int(vcapture.get(cv2.CAP_PROP_FRAME_WIDTH))
          height = int(vcapture.get(cv2.CAP_PROP_FRAME_HEIGHT))
          fps = vcapture.get(cv2.CAP_PROP_FPS)
  
          # Define codec and create video writer
          file_name = "splash_{:%Y%m%dT%H%M%S}.avi".format(datetime.datetime.now())
          vwriter = cv2.VideoWriter(file_name,
                                    cv2.VideoWriter_fourcc(*'MJPG'),
                                    fps, (width, height))
  
          count = 0
          success = True
          while success:
              print("frame: ", count)
              # Read next image
              success, image = vcapture.read()
              if success:
                  # OpenCV returns images as BGR, convert to RGB
                  image = image[..., ::-1]
                  # Detect objects
                  r = model.detect([image], verbose=0)[0]
                  # Color splash
                  splash = color_splash(image, r['masks'])
                  # RGB -> BGR to save image to video
                  splash = splash[..., ::-1]
                  # Add image to video writer
                  vwriter.write(splash)
                  count += 1
          vwriter.release()
      print("Saved to ", file_name)
  
  
  ############################################################
  #  Training
  ############################################################
  
  if __name__ == '__main__':
      import argparse
  
      # Parse command line arguments
      parser = argparse.ArgumentParser(
          description='Train Mask R-CNN to detect Fruit.')
      parser.add_argument("command",
                          metavar="<command>",
                          help="'train' or 'splash'")
      parser.add_argument('--dataset', required=False,
                          metavar="/path/to/Fruit/dataset/",
                          help='Directory of the Fruit dataset')
      parser.add_argument('--weights', required=True,
                          metavar="/path/to/weights.h5",
                          help="Path to weights .h5 file or 'coco'")
      parser.add_argument('--logs', required=False,
                          default=DEFAULT_LOGS_DIR,
                          metavar="/path/to/logs/",
                          help='Logs and checkpoints directory (default=logs/)')
      parser.add_argument('--image', required=False,
                          metavar="path or URL to image",
                          help='Image to apply the color splash effect on')
      parser.add_argument('--video', required=False,
                          metavar="path or URL to video",
                          help='Video to apply the color splash effect on')
      args = parser.parse_args()
  
      # Validate arguments
      if args.command == "train":
          assert args.dataset, "Argument --dataset is required for training"
      elif args.command == "splash":
          assert args.image or args.video,\
                 "Provide --image or --video to apply color splash"
  
      print("Weights: ", args.weights)
      print("Dataset: ", args.dataset)
      print("Logs: ", args.logs)
  
      # Configurations
      if args.command == "train":
          config = FruitConfig()
      else:
          class InferenceConfig(FruitConfig):
              # Set batch size to 1 since we'll be running inference on
              # one image at a time. Batch size = GPU_COUNT * IMAGES_PER_GPU
              GPU_COUNT = 1
              IMAGES_PER_GPU = 1
  
              POST_NMS_ROIS_INFERENCE = 2000
  
              # proved->the higher the image quality, the better the detection accuracy will be increased
              # How every, the detection speed will be slowed dramatically
              IMAGE_RESIZE_MODE = "square"
              IMAGE_MIN_DIM = 800
              IMAGE_MAX_DIM = 3520  # was 1024,
              # for 3692*5000 images: 2560 can pass, 3520pass, 3840pass[memory warning *2],4160 pass[warning*8]
              # MAX----4480 pass[many warnings]----MAX, 4800 FAILED
  
              # Non-max suppression threshold to filter RPN proposals.
              # You can increase this during training to generate more propsals.
              RPN_NMS_THRESHOLD = 0.7
  
              # Minimum probability value to accept a detected instance
              # ROIs below this threshold are skipped
              DETECTION_MIN_CONFIDENCE = 0.6
  
              # Max number of final detections
              DETECTION_MAX_INSTANCES = 200
          config = InferenceConfig()
      config.display()
  
      # Create model
      if args.command == "train":
          model = modellib.MaskRCNN(mode="training", config=config,
                                    model_dir=args.logs)
      else:
          model = modellib.MaskRCNN(mode="inference", config=config,
                                    model_dir=args.logs)
  
      # Select weights file to load
      if args.weights.lower() == "coco":
          weights_path = COCO_WEIGHTS_PATH
          # Download weights file
          if not os.path.exists(weights_path):
              utils.download_trained_weights(weights_path)
      elif args.weights.lower() == "last":
          # Find last trained weights
          weights_path = model.find_last()
      elif args.weights.lower() == "imagenet":
          # Start from ImageNet trained weights
          weights_path = model.get_imagenet_weights()
      else:
          weights_path = args.weights
  
      # Load weights
      print("Loading weights ", weights_path)
      if args.weights.lower() == "coco":
          # Exclude the last layers because they require a matching
          # number of classes
          model.load_weights(weights_path, by_name=True, exclude=[
              "mrcnn_class_logits", "mrcnn_bbox_fc",
              "mrcnn_bbox", "mrcnn_mask"])
      else:
          model.load_weights(weights_path, by_name=True)
  
      # Train or evaluate
      if args.command == "train":
          train(model)
      elif args.command == "splash":
          detect_and_color_splash(model, image_path=args.image,
                                  video_path=args.video)
      else:
          print("'{}' is not recognized. "
                "Use 'train' or 'splash'".format(args.command))