# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 2
#     language: python
#     name: python2
# ---

# +
#info
#author: <NAME>(<EMAIL>)
#Main script for performing temporal color constancy system using RCC-Net in paper:
#<NAME>, <NAME>, <NAME>, <NAME>, <NAME>
#Recurrent Color Constancy.ICCV, 2017.

#settting
import numpy as np
from pylab import *
# %matplotlib inline
plt.rcParams['figure.figsize'] = (10, 10)
plt.rcParams['image.interpolation'] = 'nearest'
plt.rcParams['image.cmap'] = 'gray'

import os
import sys
keras_path='/usr/local/lib/python2.7/dist-packages'
sys.path.insert(0,keras_path)

# %load_ext autoreload
# %autoreload 2

# +
#load the whole dataset greyball
import scipy.io as sio
path_sfucolor='/home/qian2/qian2_old/AWB/sfugreyball/'

#precrop images into this folder
path_sfucolor_all=os.path.join(path_sfucolor,'nonlinear_sfugreyball_all_crop/');
# path_sfucolor_gt=os.path.join(path_sfucolor,'sfugreyball_gt/');

#load groundtruth and image indexes, this mat file is provided.
path_sfucolor_mat=os.path.join(path_sfucolor,'nonlinear_index&gt.mat');
mat=sio.loadmat(path_sfucolor_mat)
matrix_gt_index=mat['matrix_gt_index']
matrix_illum=mat['real_rgb'];
print matrix_illum.shape
print matrix_gt_index.shape

# +
#load the deep feature of the whole dataset
path_fcn='/home/qian2/qian2_old/fcn'

#name_feature='deepfeature/greyball_all_vggfc6.mat'
name_feature='deepfeature/greyball_all_vggfc6_nonlinear_crop.npy'
path_feature=os.path.join(path_fcn,name_feature)
matrix_feature=np.load(path_feature)
print matrix_feature.shape
# -

#load the temporal dataset (index of frames of each video)
path_temporal='/home/qian2/qian2_old/temporalcolorconstancy/'
name_temporal='index_temporal_greyball_thres0.mat';
path_temporal=os.path.join(path_temporal,name_temporal);
mat=sio.loadmat(path_temporal)
matrix_index=mat['indexs']
matrix_sequencenum=mat['sequence_num']

#get absolute indexs
matrix_allindex=np.array(matrix_index,copy='True')
for i in np.arange(matrix_index.shape[1]):
    matrix_allindex[0][i]=np.arange(0,matrix_sequencenum[i],1)+np.sum(matrix_sequencenum[0:i])

print matrix_index.shape
print matrix_allindex.shape
print matrix_allindex[0,0]

del matrix_feature

from keras.models import Sequential
from keras.layers.core import Dense,Dropout,Activation,Reshape,Flatten
from keras.layers.convolutional import Convolution2D, MaxPooling2D, ZeroPadding2D
from keras.layers.embeddings import Embedding
from keras.layers.recurrent import LSTM
from keras.optimizers import SGD
from keras.objectives import cosine_proximity
from keras.objectives import mean_squared_error
from keras.objectives import mean_squared_logarithmic_error
from keras.layers.wrappers import TimeDistributed
from keras.layers.core import Merge


from keras import backend as K
def angular_loss(y_true,y_pred):
    y_true=K.l2_normalize(y_true,axis=-1)
    y_pred=K.l2_normalize(y_pred,axis=-1)
    loss=1-(K.sum(y_true*y_pred,axis=-1))
    return loss


# +
print('Build model...')

model1 = Sequential()
model1.add(Reshape((1,32,32,3),input_shape=(32,32,3)))           
model2 = Sequential()
model2.add(Reshape((1,32,32,3),input_shape=(32,32,3)))
model3 = Sequential()
model3.add(Reshape((1,32,32,3),input_shape=(32,32,3)))           

merged=Merge([model1,model2,model3],mode='concat',concat_axis=1) #(2,3,224,224)
#merged=Concatenate([model1,model2,model3],axis=1) #later we found 3-frame-long simulated sequence is just enough
model_fake = Sequential()
model_fake.add(merged)
#     model_fake.add(TimeDistributed(ZeroPadding2D((1,1),input_shape=(3,32,32))))
model_fake.add(TimeDistributed(Convolution2D(240, 1, 1, activation='sigmoid')))
model_fake.add(TimeDistributed(MaxPooling2D((8,8), strides=(8,8))))
model_fake.add(TimeDistributed(Flatten()))
#     model_fake.add(TimeDistributed(Dense(4096, activation='relu')))#(samples,max_caption_len,4096
model_fake.add(LSTM(128,activation='sigmoid',inner_activation='sigmoid',input_shape=(None,3840)))

model_real = Sequential()
model_real.add(LSTM(output_dim=128,input_dim=4096,input_length=5,activation='sigmoid', inner_activation='sigmoid'))

model = Sequential()
model.add(Merge([model_fake,model_real],mode='concat',concat_axis=1))
#model.add(Concatenate([model_fake,model_real],axis=-1))
model.add(Dropout(0.5))
model.add(Dense(64,input_shape=(128,)))
model.add(Activation('sigmoid'))
model.add(Dense(3,input_shape=(64,)))
model.add(Activation("linear"))  

model.compile(loss=angular_loss, optimizer="rmsprop")
Wsave_initial = model.get_weights()


# +
#measure the angular error
def get_angular_error(predicted,testlabel):
    norm_a=np.linalg.norm(predicted,axis=1)
    norm_b=np.linalg.norm(testlabel,axis=1)
    norm_ab=np.multiply(norm_a,norm_b)
    product_ab=np.multiply(predicted,testlabel)
    product_ab=np.sum(product_ab,axis=1)
    divide_ab=np.divide(product_ab,norm_ab)
    angu_error_ab=np.arccos(divide_ab)
    
    print np.min(angu_error_ab)*180/np.pi
    print np.mean(angu_error_ab)*180/np.pi
    print np.max(angu_error_ab)*180/np.pi
    print np.median(angu_error_ab)*180/np.pi
    return angu_error_ab

from PIL import Image
from keras.preprocessing.image import ImageDataGenerator,array_to_img,img_to_array,load_img
from mimic import MimickingGenerator
train_datagen=MimickingGenerator(rotation_range=5,zoom_range=0.5,width_shift_range=0.1,height_shift_range=0.1)
test_datagen=ImageDataGenerator(rescale=1.0);

def readimg_greyball(path_sfucolor_all,img_name):
    img_name=img_name+'.jpg'
    file_jpg=os.path.join(path_sfucolor_all,img_name)
    img=load_img(file_jpg)
    img=img.resize((32,32),Image.NEAREST)
    x=img_to_array(img)
    return x

def getmimic(indexs_img,time_range,matrix_gt_index,path_sfucolor_all,train_datagen):
    nb_samples=indexs_img.shape[0]
    X_test=np.zeros((nb_samples,time_range,32,32,3))
    for i in range(nb_samples):
        index=indexs_img[i]
        img_name=matrix_gt_index[0,index][0]
        x=readimg_greyball(path_sfucolor_all,img_name)
        x=x.reshape((1,)+x.shape) #(1,3,32,32)
        trueimg=x
        for number_merge in range(time_range):
            for batch in train_datagen.flow(x,batch_size=1):
                break
            X_test[i,number_merge,:,:,:]=batch[0,:,:,:]
        X_test[i,-1,:,:,:]=trueimg
    return X_test


# +
#training and testing in 15-folds
all_angular_error=np.empty(shape=[0])
prediction=np.empty(shape=[0,3])
time_range=3

for testid in range(15):
    print 'sequence_testing:%d' % testid
    sequence_testing=testid
    sequence_training=[i for i in range(15) if i!=sequence_testing]
    path_testdata='data_win5/nonlinear_data_fullvideo_%d.npz' % sequence_testing
    path_testclass='data_win5/label_fullvideo%d.npz' % sequence_testing
    testdata=np.load(path_testdata)['matrix_3d']
    testlabel=np.load(path_testclass)['matrix_label']
    
    indexs_img=matrix_allindex[0,testid]
    nb_samples=indexs_img.shape[0]
    testdata2= getmimic(indexs_img,time_range,matrix_gt_index,path_sfucolor_all,train_datagen)
    print testdata.shape,testdata2.shape,testlabel.shape

    
    traindata=np.empty(shape=[0,5,4096])
    traindata2=np.empty((0,time_range,32,32,3))
    trainlabel=np.empty(shape=[0,3])
    
    for i in sequence_training:
        path_data='data_win5/nonlinear_data_fullvideo_%d.npz' % i
        path_class='data_win5/label_fullvideo%d.npz' % i
        data=np.load(path_data)['matrix_3d']
        label=np.load(path_class)['matrix_label']
        indexs_img=matrix_allindex[0,i]
        data2=getmimic(indexs_img,time_range,matrix_gt_index,path_sfucolor_all,train_datagen)
        traindata=np.append(traindata,data,axis=0)
        traindata2=np.append(traindata2,data2,axis=0)
        trainlabel=np.append(trainlabel,label,axis=0)
        
        
    print traindata.shape,traindata2.shape,trainlabel.shape
    

    weights_path='data_win5/nonlinear_weights_fullvideo%d' % testid 
    if os.path.isfile(weights_path):
        model.load_weights(weights_path)
    else:
        model.set_weights(Wsave_initial)
        model.fit([traindata2[:,0,:,:,:],traindata2[:,1,:,:,:],traindata2[:,2,:,:,:],traindata], trainlabel, batch_size=128, nb_epoch=80, validation_split=0.01)
        model.save_weights(weights_path);
        
    #load weights, and one more round    
    #model.fit([traindata2[:,0,:,:,:],traindata2[:,1,:,:,:],traindata2[:,2,:,:,:],traindata], trainlabel, batch_size=128, nb_epoch=100, validation_split=0.01)
    
    predicted = model.predict([testdata2[:,0,:,:,:],testdata2[:,1,:,:,:],testdata2[:,2,:,:,:],testdata]) 
    
    all_angular_error=np.append(all_angular_error,get_angular_error(predicted,testlabel),axis=0)
    prediction=np.append(prediction,predicted,axis=0);

# -

import datetime
print datetime.date.today()
print np.min(all_angular_error)*180/np.pi
print np.mean(all_angular_error)*180/np.pi
print np.max(all_angular_error)*180/np.pi
print np.median(all_angular_error)*180/np.pi
p=0.1
biggest10_angular_error=np.sort(all_angular_error)[-np.int(all_angular_error.shape[0]*p):-1]
print np.mean(biggest10_angular_error)*180/np.pi

#results reported on paper.
print np.min(all_angular_error)*180/np.pi
print np.mean(all_angular_error)*180/np.pi
print np.max(all_angular_error)*180/np.pi
print np.median(all_angular_error)*180/np.pi
p=0.1
biggest10_angular_error=np.sort(all_angular_error)[-np.int(all_angular_error.shape[0]*p):-1]
print np.mean(biggest10_angular_error)*180/np.pi

# +
rgb_estimate=prediction
angular_error=all_angular_error
print rgb_estimate.shape
print angular_error.shape

import scipy.io as sio
sio.savemat('results_RCCnet_nonlinear_sfugreyball.mat',mdict={'rgb_estimate': rgb_estimate,'angular_error':angular_error})
# -





StarcoderdataJupyterScriptsDedupFiltered