code: #!/usr/bin/env python3
  # -*- coding: utf-8 -*-
  """
  Created on Sun Feb 25 11:42:23 2018
  
  @author: jcanete
  """
  
  import os
  import sys
  import random
  import warnings
  
  import numpy as np
  import pandas as pd
  
  import matplotlib.pyplot as plt
  
  from tqdm import tqdm
  from itertools import chain
  from skimage.io import imread, imshow, imread_collection, concatenate_images
  from skimage.transform import resize
  from skimage.morphology import label
  
  from keras.models import Model, load_model
  from keras.layers import Input
  from keras.layers.core import Lambda
  from keras.layers.convolutional import Conv2D
  from keras.callbacks import EarlyStopping, ModelCheckpoint
  from keras import backend as K
  
  from ConvModel import ConvModel
  
  import tensorflow as tf
  #from tensorflow.python.client import device_lib
  config = tf.ConfigProto()
  config.gpu_options.allow_growth=True
  sess = tf.Session(config=config)
  #sess = tf.Session(config=tf.ConfigProto(log_device_placement=True))
  
  #print (device_lib.list_local_devices())
  
  tf.test.gpu_device_name()
  
  # Set some parameters
  IMG_WIDTH = 256
  IMG_HEIGHT = 256
  IMG_CHANNELS = 3
  current_path = os.getcwd()
  dsb_data_dir = os.path.join(current_path, "Resources")
  TRAIN_PATH = os.path.join(dsb_data_dir, 'stage1_train')
  TEST_PATH = os.path.join(dsb_data_dir, 'stage1_test')
  
  warnings.filterwarnings('ignore', category=UserWarning, module='skimage')
  seed = 42
  random.seed = seed
  np.random.seed = seed
  
  # Get train and test IDs
  train_ids = next(os.walk(TRAIN_PATH))[1]
  test_ids = next(os.walk(TEST_PATH))[1]
  
  # Get and resize train images and masks
  X_train = np.zeros((len(train_ids), IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS), dtype=np.uint8)
  Y_train = np.zeros((len(train_ids), IMG_HEIGHT, IMG_WIDTH, 1), dtype=np.bool)
  print('Getting and resizing train images and masks ... ')
  sys.stdout.flush()
  for n, id_ in tqdm(enumerate(train_ids), total=len(train_ids)):
      # path = TRAIN_PATH + id_
      path = os.path.join(TRAIN_PATH, id_)
      
      img = imread(path + '/images/' + id_ + '.png')[:,:,:IMG_CHANNELS]
      img = resize(img, (IMG_HEIGHT, IMG_WIDTH), mode='constant', preserve_range=True)
      X_train[n] = img
      mask = np.zeros((IMG_HEIGHT, IMG_WIDTH, 1), dtype=np.bool)
      for mask_file in next(os.walk(path + '/masks/'))[2]:
          mask_ = imread(path + '/masks/' + mask_file)
          mask_ = np.expand_dims(resize(mask_, (IMG_HEIGHT, IMG_WIDTH), mode='constant', 
                                        preserve_range=True), axis=-1)
          mask = np.maximum(mask, mask_)
      Y_train[n] = mask
  
  # Get and resize test images
  X_test = np.zeros((len(test_ids), IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS), dtype=np.uint8)
  sizes_test = []
  print('Getting and resizing test images ... ')
  sys.stdout.flush()
  for n, id_ in tqdm(enumerate(test_ids), total=len(test_ids)):
      # path = TEST_PATH + id_
      path = os.path.join(TEST_PATH, id_)
      img = imread(path + '/images/' + id_ + '.png')[:,:,:IMG_CHANNELS]
      sizes_test.append([img.shape[0], img.shape[1]])
      img = resize(img, (IMG_HEIGHT, IMG_WIDTH), mode='constant', preserve_range=True)
      X_test[n] = img
  
  print('Done!')
  
  # Check if training data looks all right
  ix = random.randint(0, len(train_ids))
  imshow(X_train[ix])
  plt.show()
  imshow(np.squeeze(Y_train[ix]))
  plt.show()
  
  # Define IoU metric
  def mean_iou(y_true, y_pred):
      prec = []
      for t in np.arange(0.5, 0.95, 0.05):
          y_pred_ = tf.to_int32(y_pred > t)
          score, up_opt = tf.metrics.mean_iou(y_true, y_pred_, 2, y_true)
          K.get_session().run(tf.local_variables_initializer())
          with tf.control_dependencies([up_opt]):
              score = tf.identity(score)
          prec.append(score)
      return K.mean(K.stack(prec), axis=0)
  
  # Build U-Net model
  inputs = Input((IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS))
  s = Lambda(lambda x: x / 255) (inputs)
  
  conv_model = ConvModel.ConvModel(inputs, s)
  
  # Ready the model
  conv_model.u_net()
  outputs = Conv2D(1, (1, 1), activation='sigmoid') (conv_model.model)
  
  model = Model(inputs=[inputs], outputs=[outputs])
  model.compile(optimizer='nadam', loss='binary_crossentropy', metrics=[mean_iou])
  model.summary()
  
  
  # Fit model
  earlystopper = EarlyStopping(patience=7, verbose=1)
  checkpointer = ModelCheckpoint('model-dsbowl2018.h5', verbose=1, save_best_only=True)
  results = model.fit(X_train, Y_train, validation_split=0.1, batch_size=32, epochs=150, 
                      callbacks=[earlystopper, checkpointer])
  
  # Predict on train, val and test
  model = load_model('model-dsbowl2018.h5', custom_objects={'mean_iou': mean_iou})
  preds_train = model.predict(X_train[:int(X_train.shape[0]*0.9)], verbose=1)
  preds_val = model.predict(X_train[int(X_train.shape[0]*0.9):], verbose=1)
  preds_test = model.predict(X_test, verbose=1)
  
  # Threshold predictions
  preds_train_t = (preds_train > 0.5).astype(np.uint8)
  preds_val_t = (preds_val > 0.5).astype(np.uint8)
  preds_test_t = (preds_test > 0.5).astype(np.uint8)
  
  # Create list of upsampled test masks
  preds_test_upsampled = []
  for i in range(len(preds_test)):
      preds_test_upsampled.append(resize(np.squeeze(preds_test[i]), 
                                         (sizes_test[i][0], sizes_test[i][1]), 
                                         mode='constant', preserve_range=True))
  
  # Perform a sanity check on some random training samples
  ix = random.randint(0, len(preds_train_t))
  imshow(X_train[ix])
  plt.show()
  imshow(np.squeeze(Y_train[ix]))
  plt.show()
  imshow(np.squeeze(preds_train_t[ix]))
  plt.show()
  
  # Perform a sanity check on some random validation samples
  ix = random.randint(0, len(preds_val_t))
  imshow(X_train[int(X_train.shape[0]*0.9):][ix])
  plt.show()
  imshow(np.squeeze(Y_train[int(Y_train.shape[0]*0.9):][ix]))
  plt.show()
  imshow(np.squeeze(preds_val_t[ix]))
  plt.show()
  
  
  # Run-length encoding stolen from https://www.kaggle.com/rakhlin/fast-run-length-encoding-python
  def rle_encoding(x):
      dots = np.where(x.T.flatten() == 1)[0]
      run_lengths = []
      prev = -2
      for b in dots:
          if (b>prev+1): run_lengths.extend((b + 1, 0))
          run_lengths[-1] += 1
          prev = b
      return run_lengths
  
  def prob_to_rles(x, cutoff=0.5):
      lab_img = label(x > cutoff)
      for i in range(1, lab_img.max() + 1):
          yield rle_encoding(lab_img == i)
  
  
  new_test_ids = []
  rles = []
  for n, id_ in enumerate(test_ids):
      rle = list(prob_to_rles(preds_test_upsampled[n]))
      rles.extend(rle)
      new_test_ids.extend([id_] * len(rle))
  
  
  # Create submission DataFrame
  sub = pd.DataFrame()
  sub['ImageId'] = new_test_ids
  sub['EncodedPixels'] = pd.Series(rles).apply(lambda x: ' '.join(str(y) for y in x))
  sub.to_csv('sub-dsbowl2018-1.csv', index=False)
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
  
apis: tensorflow.metrics.mean_iou numpy.maximum pandas.Series tensorflow.test.gpu_device_name tensorflow.local_variables_initializer tensorflow.control_dependencies numpy.arange numpy.squeeze tensorflow.identity pandas.DataFrame tensorflow.ConfigProto tensorflow.Session tensorflow.to_int32 matplotlib.pyplot.show numpy.zeros
api_extract: [(36, 'tensorflow.ConfigProto', 'tf.ConfigProto', ([], {}), True, 'import tensorflow as tf\n'), (38, 'tensorflow.Session', 'tf.Session', ([], {'config': 'config'}), True, 'import tensorflow as tf\n'), (43, 'tensorflow.test.gpu_device_name', 'tf.test.gpu_device_name', ([], {}), True, 'import tensorflow as tf\n'), (49, 'os.getcwd', 'os.getcwd', ([], {}), False, 'import os\n'), (50, 'os.path.join', 'os.path.join', (['current_path', '"""Resources"""'], {}), False, 'import os\n'), (51, 'os.path.join', 'os.path.join', (['dsb_data_dir', '"""stage1_train"""'], {}), False, 'import os\n'), (52, 'os.path.join', 'os.path.join', (['dsb_data_dir', '"""stage1_test"""'], {}), False, 'import os\n'), (54, 'warnings.filterwarnings', 'warnings.filterwarnings', (['"""ignore"""'], {'category': 'UserWarning', 'module': '"""skimage"""'}), False, 'import warnings\n'), (67, 'sys.stdout.flush', 'sys.stdout.flush', ([], {}), False, 'import sys\n'), (87, 'sys.stdout.flush', 'sys.stdout.flush', ([], {}), False, 'import sys\n'), (100, 'skimage.io.imshow', 'imshow', (['X_train[ix]'], {}), False, 'from skimage.io import imread, imshow, imread_collection, concatenate_images\n'), (101, 'matplotlib.pyplot.show', 'plt.show', ([], {}), True, 'import matplotlib.pyplot as plt\n'), (103, 'matplotlib.pyplot.show', 'plt.show', ([], {}), True, 'import matplotlib.pyplot as plt\n'), (118, 'keras.layers.Input', 'Input', (['(IMG_HEIGHT, IMG_WIDTH, IMG_CHANNELS)'], {}), False, 'from keras.layers import Input\n'), (121, 'ConvModel.ConvModel.ConvModel', 'ConvModel.ConvModel', (['inputs', 's'], {}), False, 'from ConvModel import ConvModel\n'), (127, 'keras.models.Model', 'Model', ([], {'inputs': '[inputs]', 'outputs': '[outputs]'}), False, 'from keras.models import Model, load_model\n'), (133, 'keras.callbacks.EarlyStopping', 'EarlyStopping', ([], {'patience': '(7)', 'verbose': '(1)'}), False, 'from keras.callbacks import EarlyStopping, ModelCheckpoint\n'), (134, 'keras.callbacks.ModelCheckpoint', 'ModelCheckpoint', (['"""model-dsbowl2018.h5"""'], {'verbose': '(1)', 'save_best_only': '(True)'}), False, 'from keras.callbacks import EarlyStopping, ModelCheckpoint\n'), (139, 'keras.models.load_model', 'load_model', (['"""model-dsbowl2018.h5"""'], {'custom_objects': "{'mean_iou': mean_iou}"}), False, 'from keras.models import Model, load_model\n'), (158, 'skimage.io.imshow', 'imshow', (['X_train[ix]'], {}), False, 'from skimage.io import imread, imshow, imread_collection, concatenate_images\n'), (159, 'matplotlib.pyplot.show', 'plt.show', ([], {}), True, 'import matplotlib.pyplot as plt\n'), (161, 'matplotlib.pyplot.show', 'plt.show', ([], {}), True, 'import matplotlib.pyplot as plt\n'), (163, 'matplotlib.pyplot.show', 'plt.show', ([], {}), True, 'import matplotlib.pyplot as plt\n'), (168, 'matplotlib.pyplot.show', 'plt.show', ([], {}), True, 'import matplotlib.pyplot as plt\n'), (170, 'matplotlib.pyplot.show', 'plt.show', ([], {}), True, 'import matplotlib.pyplot as plt\n'), (172, 'matplotlib.pyplot.show', 'plt.show', ([], {}), True, 'import matplotlib.pyplot as plt\n'), (201, 'pandas.DataFrame', 'pd.DataFrame', ([], {}), True, 'import pandas as pd\n'), (70, 'os.path.join', 'os.path.join', (['TRAIN_PATH', 'id_'], {}), False, 'import os\n'), (73, 'skimage.transform.resize', 'resize', (['img', '(IMG_HEIGHT, IMG_WIDTH)'], {'mode': '"""constant"""', 'preserve_range': '(True)'}), False, 'from skimage.transform import resize\n'), (75, 'numpy.zeros', 'np.zeros', (['(IMG_HEIGHT, IMG_WIDTH, 1)'], {'dtype': 'np.bool'}), True, 'import numpy as np\n'), (90, 'os.path.join', 'os.path.join', (['TEST_PATH', 'id_'], {}), False, 'import os\n'), (93, 'skimage.transform.resize', 'resize', (['img', '(IMG_HEIGHT, IMG_WIDTH)'], {'mode': '"""constant"""', 'preserve_range': '(True)'}), False, 'from skimage.transform import resize\n'), (102, 'numpy.squeeze', 'np.squeeze', (['Y_train[ix]'], {}), True, 'import numpy as np\n'), (108, 'numpy.arange', 'np.arange', (['(0.5)', '(0.95)', '(0.05)'], {}), True, 'import numpy as np\n'), (119, 'keras.layers.core.Lambda', 'Lambda', (['(lambda x: x / 255)'], {}), False, 'from keras.layers.core import Lambda\n'), (125, 'keras.layers.convolutional.Conv2D', 'Conv2D', (['(1)', '(1, 1)'], {'activation': '"""sigmoid"""'}), False, 'from keras.layers.convolutional import Conv2D\n'), (160, 'numpy.squeeze', 'np.squeeze', (['Y_train[ix]'], {}), True, 'import numpy as np\n'), (162, 'numpy.squeeze', 'np.squeeze', (['preds_train_t[ix]'], {}), True, 'import numpy as np\n'), (171, 'numpy.squeeze', 'np.squeeze', (['preds_val_t[ix]'], {}), True, 'import numpy as np\n'), (187, 'skimage.morphology.label', 'label', (['(x > cutoff)'], {}), False, 'from skimage.morphology import label\n'), (60, 'os.walk', 'os.walk', (['TRAIN_PATH'], {}), False, 'import os\n'), (61, 'os.walk', 'os.walk', (['TEST_PATH'], {}), False, 'import os\n'), (72, 'skimage.io.imread', 'imread', (["(path + '/images/' + id_ + '.png')"], {}), False, 'from skimage.io import imread, imshow, imread_collection, concatenate_images\n'), (77, 'skimage.io.imread', 'imread', (["(path + '/masks/' + mask_file)"], {}), False, 'from skimage.io import imread, imshow, imread_collection, concatenate_images\n'), (80, 'numpy.maximum', 'np.maximum', (['mask', 'mask_'], {}), True, 'import numpy as np\n'), (91, 'skimage.io.imread', 'imread', (["(path + '/images/' + id_ + '.png')"], {}), False, 'from skimage.io import imread, imshow, imread_collection, concatenate_images\n'), (109, 'tensorflow.to_int32', 'tf.to_int32', (['(y_pred > t)'], {}), True, 'import tensorflow as tf\n'), (110, 'tensorflow.metrics.mean_iou', 'tf.metrics.mean_iou', (['y_true', 'y_pred_', '(2)', 'y_true'], {}), True, 'import tensorflow as tf\n'), (115, 'keras.backend.stack', 'K.stack', (['prec'], {}), True, 'from keras import backend as K\n'), (203, 'pandas.Series', 'pd.Series', (['rles'], {}), True, 'import pandas as pd\n'), (76, 'os.walk', 'os.walk', (["(path + '/masks/')"], {}), False, 'import os\n'), (78, 'skimage.transform.resize', 'resize', (['mask_', '(IMG_HEIGHT, IMG_WIDTH)'], {'mode': '"""constant"""', 'preserve_range': '(True)'}), False, 'from skimage.transform import resize\n'), (111, 'tensorflow.local_variables_initializer', 'tf.local_variables_initializer', ([], {}), True, 'import tensorflow as tf\n'), (112, 'tensorflow.control_dependencies', 'tf.control_dependencies', (['[up_opt]'], {}), True, 'import tensorflow as tf\n'), (113, 'tensorflow.identity', 'tf.identity', (['score'], {}), True, 'import tensorflow as tf\n'), (152, 'numpy.squeeze', 'np.squeeze', (['preds_test[i]'], {}), True, 'import numpy as np\n'), (111, 'keras.backend.get_session', 'K.get_session', ([], {}), True, 'from keras import backend as K\n')]