text: ## based on https://github.com/matterport/Mask_RCNN
  
  
  import sys
  import os
  import logging
  import math
  import random
  import json
  import numpy as np
  import scipy
  import skimage.color
  import skimage.io
  import skimage.transform
  import urllib.request
  import shutil
  import warnings
  from distutils.version import LooseVersion
  from PIL import Image, ImageDraw
  
  class Dataset(object):
      """The base class for dataset classes.
      To use it, create a new class that adds functions specific to the dataset
      you want to use. For example:
      class CatsAndDogsDataset(Dataset):
          def load_cats_and_dogs(self):
              ...
          def load_mask(self, image_id):
              ...
          def image_reference(self, image_id):
              ...
      See COCODataset and ShapesDataset as examples.
      """
  
      def __init__(self, class_map=None):
          self._image_ids = []
          self.image_info = []
          # Background is always the first class
          self.class_info = [{"source": "", "id": 0, "name": "BG"}]
          self.source_class_ids = {}
  
      def add_class(self, source, class_id, class_name):
          assert "." not in source, "Source name cannot contain a dot"
          # Does the class exist already?
          for info in self.class_info:
              if info['source'] == source and info["id"] == class_id:
                  # source.class_id combination already available, skip
                  return
          # Add the class
          self.class_info.append({
              "source": source,
              "id": class_id,
              "name": class_name,
          })
  
      def add_image(self, source, image_id, path, **kwargs):
          image_info = {
              "id": image_id,
              "source": source,
              "path": path,
          }
          image_info.update(kwargs)
          self.image_info.append(image_info)
  
      def image_reference(self, image_id):
          """Return a link to the image in its source Website or details about
          the image that help looking it up or debugging it.
          Override for your dataset, but pass to this function
          if you encounter images not in your dataset.
          """
          return ""
  
      def prepare(self, class_map=None):
          """Prepares the Dataset class for use.
          TODO: class map is not supported yet. When done, it should handle mapping
                classes from different datasets to the same class ID.
          """
  
          def clean_name(name):
              """Returns a shorter version of object names for cleaner display."""
              return ",".join(name.split(",")[:1])
  
          # Build (or rebuild) everything else from the info dicts.
          self.num_classes = len(self.class_info)
          self.class_ids = np.arange(self.num_classes)
          self.class_names = [clean_name(c["name"]) for c in self.class_info]
          self.num_images = len(self.image_info)
          self._image_ids = np.arange(self.num_images)
  
          # Mapping from source class and image IDs to internal IDs
          self.class_from_source_map = {"{}.{}".format(info['source'], info['id']): id
                                        for info, id in zip(self.class_info, self.class_ids)}
          self.image_from_source_map = {"{}.{}".format(info['source'], info['id']): id
                                        for info, id in zip(self.image_info, self.image_ids)}
  
          # Map sources to class_ids they support
          self.sources = list(set([i['source'] for i in self.class_info]))
          self.source_class_ids = {}
          # Loop over datasets
          for source in self.sources:
              self.source_class_ids[source] = []
              # Find classes that belong to this dataset
              for i, info in enumerate(self.class_info):
                  # Include BG class in all datasets
                  if i == 0 or source == info['source']:
                      self.source_class_ids[source].append(i)
  
      def map_source_class_id(self, source_class_id):
          """Takes a source class ID and returns the int class ID assigned to it.
          For example:
          dataset.map_source_class_id("coco.12") -> 23
          """
          return self.class_from_source_map[source_class_id]
  
      def get_source_class_id(self, class_id, source):
          """Map an internal class ID to the corresponding class ID in the source dataset."""
          info = self.class_info[class_id]
          assert info['source'] == source
          return info['id']
  
      @property
      def image_ids(self):
          return self._image_ids
  
      def source_image_link(self, image_id):
          """Returns the path or URL to the image.
          Override this to return a URL to the image if it's available online for easy
          debugging.
          """
          return self.image_info[image_id]["path"]
  
      def load_image(self, image_id):
          """Load the specified image and return a [H,W,3] Numpy array.
          """
          # Load image
          image = skimage.io.imread(self.image_info[image_id]['path'])
          # If grayscale. Convert to RGB for consistency.
          if image.ndim != 3:
              image = skimage.color.gray2rgb(image)
          # If has an alpha channel, remove it for consistency
          if image.shape[-1] == 4:
              image = image[..., :3]
          return image
  
      def load_mask(self, image_id):
          """Load instance masks for the given image.
          Different datasets use different ways to store masks. Override this
          method to load instance masks and return them in the form of am
          array of binary masks of shape [height, width, instances].
          Returns:
              masks: A bool array of shape [height, width, instance count] with
                  a binary mask per instance.
              class_ids: a 1D array of class IDs of the instance masks.
          """
          # Override this function to load a mask from your dataset.
          # Otherwise, it returns an empty mask.
          logging.warning("You are using the default load_mask(), maybe you need to define your own one.")
          mask = np.empty([0, 0, 0])
          class_ids = np.empty([0], np.int32)
          return mask, class_ids
  
  class CocoLikeDataset(Dataset):
      """ Generates a COCO-like dataset, i.e. an image dataset annotated in the style of the COCO dataset.
          See http://cocodataset.org/#home for more information.
      """
      def load_data(self, annotation_json, images_dir):
          """ Load the coco-like dataset from json
          Args:
              annotation_json: The path to the coco annotations json file
              images_dir: The directory holding the images referred to by the json file
          """
          # Load json from file
          json_file = open(annotation_json)
          coco_json = json.load(json_file)
          json_file.close()
          
          # Add the class names using the base method from utils.Dataset
          source_name = "coco_like"
          for category in coco_json['categories']:
              class_id = category['id']
              class_name = category['name']
              if class_id < 1:
                  print('Error: Class id for "{}" cannot be less than one. (0 is reserved for the background)'.format(class_name))
                  return
              
              self.add_class(source_name, class_id, class_name)
          
          # Get all annotations
          annotations = {}
          for annotation in coco_json['annotations']:
              image_id = annotation['image_id']
              if image_id not in annotations:
                  annotations[image_id] = []
              annotations[image_id].append(annotation)
          
          # Get all images and add them to the dataset
          seen_images = {}
          for image in coco_json['images']:
              image_id = image['id']
              if image_id in seen_images:
                  print("Warning: Skipping duplicate image id: {}".format(image))
              else:
                  seen_images[image_id] = image
                  try:
                      image_file_name = image['file_name']
                      image_width = image['width']
                      image_height = image['height']
                  except KeyError as key:
                      print("Warning: Skipping image (id: {}) with missing key: {}".format(image_id, key))
                  
                  image_path = os.path.abspath(os.path.join(images_dir, image_file_name))
                  image_annotations = annotations[image_id]
                  
                  # Add the image using the base method from utils.Dataset
                  self.add_image(
                      source=source_name,
                      image_id=image_id,
                      path=image_path,
                      width=image_width,
                      height=image_height,
                      annotations=image_annotations
                  )
                  
      def load_mask(self, image_id):
          """ Load instance masks for the given image.
          MaskRCNN expects masks in the form of a bitmap [height, width, instances].
          Args:
              image_id: The id of the image to load masks for
          Returns:
              masks: A bool array of shape [height, width, instance count] with
                  one mask per instance.
              class_ids: a 1D array of class IDs of the instance masks.
          """
          image_info = self.image_info[image_id]
          annotations = image_info['annotations']
          instance_masks = []
          class_ids = []
          
          for annotation in annotations:
              class_id = annotation['category_id']
              mask = Image.new('1', (image_info['width'], image_info['height']))
              mask_draw = ImageDraw.ImageDraw(mask, '1')
              for segmentation in annotation['segmentation']:
                  mask_draw.polygon(segmentation, fill=1)
                  bool_array = np.array(mask) > 0
                  instance_masks.append(bool_array)
                  class_ids.append(class_id)
  
          mask = np.dstack(instance_masks)
          class_ids = np.array(class_ids, dtype=np.int32)
          
          return mask, class_ids
  
  