# Put these at the top of every notebook, to get automatic reloading and inline plotting
%reload_ext autoreload
%autoreload 2
%matplotlib inline

# This file contains all the main external libs we'll use
import numpy as np
import pandas as pd
from fastai.imports import *
from sklearn.model_selection import train_test_split
from fastai.models.cifar10.senet import SENet
from fastai.imports import *
from fastai.transforms import *
from fastai.conv_learner import *
from fastai.model import *
from fastai.dataset import *
from fastai.sgdr import *
from fastai.plots import *

! ls data/processed/

path = "data/processed/"

train = pd.read_json(f'{path}train.json')

test = pd.read_json(f'{path}test.json')

train[:2]

len(train.iloc[4][1])

train.inc_angle = train.inc_angle.apply(lambda x: np.nan if x == 'na' else x)
test.inc_angle = test.inc_angle.apply(lambda x: np.nan if x == 'na' else x)

img1 = train.loc[0,['band_1','band_2']]

img1

img1 = np.stack([img1['band_1'], img1['band_2']], -1).reshape(75,75,2)

"""
Explanation: Deep learning using fastai library
(https://github.com/fastai/courses)
This is a beginner level code (can be learnt in 2 classes of deep learning taught by Jeremy Howard) and we can get decent results by only tuning learning rate and training until model overfits.
I have used pretrained resnet34 model based on Imagenet data for this
Steps to use fastai library --

git clone https://github.com/fastai/fastai  
cd fastai  
conda create -n fastai python=3.6 anaconda  
conda env update  
source activate fastai  

This kernel is specifically is for Beginners who want's to experiment building CNN using fastai (on the top of pytorch). By using this kernel, you can expect to get good score and also learn fastai. Fastai has made building deep neural networks very easy.
End of explanation
"""


plt.imshow(img1[:,:,1])

"""
Explanation: Below picture is not an iceberg
End of explanation
"""


def color_composite(data):
    rgb_arrays = []
    for i, row in data.iterrows():
        band_1 = np.array(row['band_1']).reshape(75, 75)
        band_2 = np.array(row['band_2']).reshape(75, 75)
        band_3 = band_1 / band_2

        r = (band_1 + abs(band_1.min())) / np.max((band_1 + abs(band_1.min())))
        g = (band_2 + abs(band_2.min())) / np.max((band_2 + abs(band_2.min())))
        b = (band_3 + abs(band_3.min())) / np.max((band_3 + abs(band_3.min())))
        
#         r = ((band_1 - np.mean(band_1)) / (np.max(band_1) - np.min(band_1))) 
#         g = ((band_2 - np.mean(band_2)) / (np.max(band_2) - np.min(band_2))) 
#         b = ((band_3 - np.mean(band_3)) / (np.max(band_3) - np.min(band_3)))

        rgb = np.dstack((r, g, b))
        rgb_arrays.append(rgb)
    return np.array(rgb_arrays)

# Trained with data about rgb
rgb_train = color_composite(train)
rgb_train.shape

# Test with data about rgb
rgb_test = color_composite(test)
rgb_test.shape

"""
Explanation: Get rgb of image using color composite function
Thanks to MadScientist for color composite.
Here is the kernal -- 
https://www.kaggle.com/keremt/getting-color-composites
End of explanation
"""


# look at random ships
print('Looking at random ships')
ships = np.random.choice(np.where(train.is_iceberg ==0)[0], 9)
fig = plt.figure(1,figsize=(12,12))
for i in range(9):
    ax = fig.add_subplot(3,3,i+1)
    arr = rgb_train[ships[i], :, :]
    ax.imshow(arr)
    
plt.show()

# look at random iceberges
print('Looking at random icebergs')
ice = np.random.choice(np.where(train.is_iceberg ==1)[0], 9)
fig = plt.figure(200,figsize=(12,12))
for i in range(9):
    ax = fig.add_subplot(3,3,i+1)
    arr = rgb_train[ice[i], :, :]
    ax.imshow(arr)
    
plt.show()

"""
Explanation: Exploring images before training CNN model
End of explanation
"""


# # making directories for training resnet (as it need files to be in right dir)

os.makedirs(f'{path}composites', exist_ok= True)
os.makedirs(f'{path}composites/train', exist_ok=True)
os.makedirs(f'{path}composites/valid', exist_ok=True)
os.makedirs(f'{path}composites/test', exist_ok=True)

dir_list = [f'{path}composites/train', f'{path}composites/valid']

for i in dir_list:
    os.makedirs(f'{i}/ship')
    os.makedirs(f'{i}/iceberg')

"""
Explanation: Observation from images --
• Ships have a trace of bright lights around them which will be taken as a feature in CNN
• Ships are more consistent with shapes
• Icebergs shape vary more than ships
saving images in directories (train, valid, test)
End of explanation
"""


! ls {path}composites

# split
train_y, valid_y = train_test_split(train.is_iceberg, test_size=0.10)

train_iceberg_index, train_ship_index, valid_iceberg_index, valid_ship_index  = train_y[train_y==1].index, train_y[train_y==0].index, valid_y[valid_y==1].index, valid_y[valid_y==0].index


#save train images
for idx in train_iceberg_index:
    img = rgb_train[idx]
    plt.imsave(f'{path}/composites/train/iceberg/' + str(idx) + '.png',  img)

for idx in train_ship_index:
    img = rgb_train[idx]
    plt.imsave(f'{path}/composites/train/ship/' + str(idx) + '.png',  img)    
    
#save valid images
for idx in valid_iceberg_index:
    img = rgb_train[idx]
    plt.imsave(f'{path}/composites/valid/iceberg/' + str(idx) + '.png',  img)

for idx in valid_ship_index:
    img = rgb_train[idx]
    plt.imsave(f'{path}/composites/valid/ship/' + str(idx) + '.png',  img)

#save test images
for idx in range(len(test)):
    img = rgb_test[idx]
    plt.imsave(f'{path}/composites/test/' + str(idx) + '.png',  img)

"""
Explanation: The reason of converting these images to .png is that the pretrained ConvLearner that I am going to call takes image as input
End of explanation
"""


path2 = 'data/processed/composites/'

"""
Explanation: let's check directory where files are saved. Ok we have train, test and valid directories
I can start from here from next time
let's train first resnet model using fastai
End of explanation
"""


files = !ls {path2}valid/ship | head

img = plt.imread(f'{path2}valid/ship/{files[0]}')
plt.imshow(img)

! ls {path2}

"""
Explanation: let's look at a random ships now (from png) to make sure images are saved in directory
End of explanation
"""


def get_data(sz, bs):
    tfms = tfms_from_model(arch, sz, aug_tfms=transforms_top_down, max_zoom=1.00)
    data = ImageClassifierData.from_paths(path2, test_name = 'test', bs = bs,
                                          tfms = tfms)
    return data 

arch=resnet34
sz = 75  # because our image size is 75*75
bs = 16 # because default batch size of 64 was not giving good converging loss

data = get_data(sz, bs)

data = data.resize(int(sz*1.5), 'tmp')

learn = ConvLearner.pretrained(arch, data, precompute=False) 

lrf = learn.lr_find()

learn.sched.plot_lr()

learn.sched.plot() 

"""
Explanation: Finding learning rate using lr finder
One of the most sensitive hyperparameter for deep learning is learning rate. Finding a good learning rate is most important step to make a good model (without over or underfitting)
End of explanation
"""


lr = 0.001
learn.unfreeze()
learn.bn_freeze(False)

"""
Explanation: Now, this plot is important to decide a good learning rate. We will not decide the learning rate at the lowest loss, which might sound confusing. But the catch is that we are going to do differential annealing to activate our layers which take varying learning rates. And the rate we chose here is going to be the maximum rate. So, we will chose some rate just before it bottoms up where loss is still falling.
Minimum is at 10^-2, I would chose 10^-3 is LR
Stochastic descent with restart
End of explanation
"""


learn.fit(lr, 3, cycle_len=1, cycle_mult=2) # precompute was false # first fit

# stochastic descent with restart
learn.sched.plot_lr()

learn.sched.plot_loss()

learn.fit(lr,4, cycle_len=1, cycle_mult=2) # precompute was false # first fit

# stochastic descent with restart
learn.sched.plot_lr()

learn.sched.plot_loss()

"""
Explanation: epochs = 3
cycle length = 1
cycle multiple = 2
End of explanation
"""


lr = 0.01

lrs=np.array([lr/9,lr/3,lr]) # use lr/100 and lr/10 respectively if images would have been larger in sizes

%time learn.fit(lrs, 5, cycle_len=1, cycle_mult=2, cycle_save_name='resnet50')

learn.sched.plot_loss()

%time learn.fit(lrs, 4, cycle_len=1, cycle_mult=2)

learn.sched.plot_loss()

"""
Explanation: Loss is decreasing with increaseing iterations
Fine tuning and differential annealing
Since, the images that we have do not have very clear and sharp features which might be present in top layers of our pre-trained neural network. Let's unfreeze our layers and calculate activations
End of explanation
"""


# to check validation accuray
log_preds,y = learn.TTA()
accuracy(log_preds,y)

%time learn.fit(lrs,5 , cycle_len=1, cycle_mult=2)

"""
Explanation: Do Train It again (atleast 2 times)
End of explanation
"""


# from here we know that 'icebergs' is label 0 and 'ships' is label 1.
data.classes

# this gives prediction for validation set. Predictions are in log scale
log_preds = learn.TTA(is_test=True) # If need TTA
#log_preds = learn.predict(is_test=True) # if don't need TTA
log_preds

probs_submit1 = np.exp(log_preds[0][:,0]) 
probs_submit1[:]

# getting ids from test list
id_raw = data.test_dl.dataset.fnames
id_raw[1]


# using regex to take numbers from ids
id_pro = result_array = np.empty((0, len(id_raw)))
for i in range(len(id_raw)):
    stuff = int(re.findall(r'\d+', id_raw[i])[0])
    #print(type(str(stuff)))
    id_pro = np.append(id_pro,int(stuff))



id_pro_list = []
for i in id_pro:
    id_pro_list.append(int(i))
id_pro_list[:4]

# joining id and probability
d = {'index': id_pro_list, 'is_iceberg': probs_submit1}
submit1_df = pd.DataFrame(data=d)

id_ = test['id']
id_pd = pd.DataFrame({'id': id_} )
submit1_df_sorted = submit1_df.sort_values('index')
submit1_df_sorted2 = pd.concat([id_, submit1_df_sorted.set_index('index')], axis = 1)
submit1_df_sorted2.dtypes
submit1_df.dtypes


d['is_iceberg1']

l=[]
for i in range(len(d['is_iceberg1'])):
    l.append(d['is_iceberg1'][i][0])

len(l)

d_new = {'index': id_pro_list, 'is_iceberg': l}
d_new['is_iceberg']

submit1_df = pd.DataFrame(data=d_new)

id_ = test['id']
id_pd = pd.DataFrame({'id': id_} )

submit1_df_sorted = submit1_df.sort_values('index')
submit1_df_sorted2 = pd.concat([id_, submit1_df_sorted.set_index('index')], axis = 1)

submit1_df_sorted2.dtypes

submit1_df.dtypes

#submit1_df.id = submit1_df.id.astype(str)

submit1_df_sorted2.to_csv('data/processed/resnet50.csv', index = False)

! head -5 data/processed/resnet50.csv

submit_check = pd.read_csv("data/processed/resnet50.csv")

submit_check.dtypes

submit_check[:10]

submit_check['is_iceberg']

"""
Explanation: TTA (Test Time Augmentation) simply makes predictions not just on the images in your validation set, but also makes predictions on a number of randomly augmented versions of them 
Predictions
End of explanation
"""


log_pred1 = learn.TTA() # If need TTA

preds = np.argmax(log_pred1[0], axis=1)  # from log probabilities to 0 or 1
probs = np.exp(log_pred1[0][:,1])        # pr(ship)

#probs = submit_check['is_iceberg']
#submit_check['C'] = np.where(submit_check['is_iceberg'] >= 0.5,1, 0)
#preds = np.array(submit_check['C'])



def rand_by_mask(mask): return np.random.choice(np.where(mask)[0], 4, replace=False)
def rand_by_correct(is_correct): return rand_by_mask((preds == data.val_y)==is_correct)

def plot_val_with_title(idxs, title):
    imgs = np.stack([data.val_ds[x][0] for x in idxs])
    title_probs = [probs[x] for x in idxs]
    print(title)
    return plots(data.val_ds.denorm(imgs), rows=1, titles=title_probs)

def plots(ims, figsize=(12,6), rows=1, titles=None):
    f = plt.figure(figsize=figsize)
    for i in range(len(ims)):
        sp = f.add_subplot(rows, len(ims)//rows, i+1)
        sp.axis('Off')
        if titles is not None: sp.set_title(titles[i], fontsize=16)
        plt.imshow(ims[i])

def load_img_id(ds, idx): return np.array(PIL.Image.open(path2+ds.fnames[idx]))

def plot_val_with_title(idxs, title):
    imgs = [load_img_id(data.val_ds,x) for x in idxs]
    title_probs = [probs[x] for x in idxs]
    print(title)
    return plots(imgs, rows=1, titles=title_probs, figsize=(16,8))

# 1. A few correct labels at random
plot_val_with_title(rand_by_correct(True), "Correctly classified")

# 2. A few incorrect labels at random
plot_val_with_title(rand_by_correct(False), "Incorrectly classified")

def most_by_mask(mask, mult):
    idxs = np.where(mask)[0]
    return idxs[np.argsort(mult * probs[idxs])[:4]]

def most_by_correct(y, is_correct): 
    mult = -1 if (y==1)==is_correct else 1
    return most_by_mask((preds == data.val_y)==is_correct & (data.val_y == y), mult)

plot_val_with_title(most_by_correct(0, True), "Most correct icebergs")

plot_val_with_title(most_by_correct(1, True), "Most correct ships")

"""
Explanation: scp submit1 to local machine to upload on kaggle
Post analysis
Looking at correctly, incorrectly classified images
Analyzing results: looking at pictures
As well as looking at the overall metrics, it's also a good idea to look at examples of each of:
* A few correct labels at random
* A few incorrect labels at random
* The most correct labels of each class (ie those with highest probability that are correct)
* The most incorrect labels of each class (ie those with highest probability that are incorrect)
* The most uncertain labels (ie those with probability closest to 0.5).
End of explanation
"""


!pwd

!ls

from fastai.models.cifar10.senet import SENet34

bm = BasicModel(SENet34().cuda(),name='iceberg_34x34')

arch = resnet34
sz = 75  # because our image size is 75*75
bs = 16 # because default batch size of 64 was not giving good converging loss

data=get_data(32,bs)

learn = ConvLearner(data,bm)

"""
Explanation: SENet34
End of explanation
"""


