"""
# ライブラリのインポート
"""
import numpy as np 
import pandas as pd
import datetime
import random
import glob
import cv2
import os
from sklearn.model_selection import train_test_split, KFold, cross_val_score
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import StratifiedKFold
import tensorflow as tf
from tensorflow import keras
from tensorflow.keras.models import Sequential, Model
from tensorflow.keras.layers import BatchNormalization, Activation, Dropout, Dense
from tensorflow.keras.optimizers import Adam
from tensorflow.keras.layers import BatchNormalization,Activation,Dropout,Dense,concatenate,Input
from tensorflow.keras.utils import plot_model
from tensorflow.keras.layers import Flatten, Conv2D, MaxPooling2D,GlobalAveragePooling2D
from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint, Callback, ReduceLROnPlateau
from tensorflow.keras import layers
from tensorflow.keras import Input
from tensorflow.keras import regularizers
from keras.applications.resnet50 import ResNet50
import matplotlib.pyplot as plt
%matplotlib inline
from category_encoders import OrdinalEncoder, OneHotEncoder, TargetEncoder
from tensorflow.keras.applications import VGG16, VGG19

def seed_everything(seed):
    random.seed(seed)
    os.environ['PYTHONHASHSEED'] = str(seed)
    np.random.seed(seed)
    tf.random.set_seed(seed)

# 乱数シード固定
seed_everything(2020)

# for dirname, _, filenames in os.walk('/kaggle/input'):
#     for filename in filenames:
#         print(os.path.join(dirname, filename))
"""
# 数値データの読み込み
"""
train = pd.read_csv('/kaggle/input/5th-datarobot-ai-academy-deep-learning/train.csv')
train = train.sort_values('id')
test = pd.read_csv('/kaggle/input/5th-datarobot-ai-academy-deep-learning/test.csv')
test = test.sort_values('id')
#display(train.shape), display(test.shape)
#display(train.head()), display(test.head())
#提出ファイル
submission = pd.read_csv('/kaggle/input/5th-datarobot-ai-academy-deep-learning/sample_submission.csv', index_col=0)
#display(submission.head())
cols=['bedrooms','bathrooms','area','zipcode']
target=['price']
#欠測値
train[cols]=train[cols].fillna(-99)
test[cols] = test[cols].fillna(-99)
# area
train['area']=train['area'].astype(str).str[:1]
test['area']=test['area'].astype(str).str[:1]

temp = train.groupby(['area'],as_index=False)[target].mean()
temp = temp.rename(columns={'price': 'area_price'})

train_1 = pd.merge(train, temp, on='area', how='left')
train_1 = train_1.drop('area', axis=1)
train = train_1.rename(columns={'area_price': 'area'})

test_1 = pd.merge(test, temp, on='area', how='left')
test_1 = test_1.drop('area', axis=1)
test = test_1.rename(columns={'area_price': 'area'})

#display(train.head())
#display(test.head())
# zipcode
train['zipcode']=train['zipcode'].astype(str).str[:1]
test['zipcode']=test['zipcode'].astype(str).str[:1]

temp = train.groupby(['zipcode'],as_index=False)[target].mean()
temp = temp.rename(columns={'price': 'zipcode_price'})

train_1 = pd.merge(train, temp, on='zipcode', how='left')
train_1 = train_1.drop('zipcode', axis=1)
train = train_1.rename(columns={'zipcode_price': 'zipcode'})

test_1 = pd.merge(test, temp, on='zipcode', how='left')
test_1 = test_1.drop('zipcode', axis=1)
test = test_1.rename(columns={'zipcode_price': 'zipcode'})

#display(train.head()), display(test.head())
train[cols]=train[cols].fillna(-99)
test[cols] = test[cols].fillna(-99)

# 正規化
scaler = StandardScaler()
#X_all = pd.concat([train, test], axis=0)
train[cols] = scaler.fit_transform(train[cols])
test[cols] = scaler.fit_transform(test[cols])
#train = X_all.iloc[:train.shape[0], :]
#test = X_all.iloc[train.shape[0]:, :]
#display(train.head())
#display(test.head())
#欠測値
train[cols]=train[cols].fillna(-99)
test[cols] = test[cols].fillna(-99)
"""
# 画像処理
"""
#画像を結合して読み込み
def load_images_unit(df,inputPath,size):
    images = []
    for i in df['id']:
        basePath0 = os.path.sep.join([inputPath, "{}_{}*".format(i,'bathroom')])
        basePath1 = os.path.sep.join([inputPath, "{}_{}*".format(i,'bedroom')])
        basePath2 = os.path.sep.join([inputPath, "{}_{}*".format(i,'frontal')])
        basePath3 = os.path.sep.join([inputPath, "{}_{}*".format(i,'kitchen')])
        housePaths0 = sorted(list(glob.glob(basePath0)))
        housePaths1 = sorted(list(glob.glob(basePath1)))
        housePaths2 = sorted(list(glob.glob(basePath2)))
        housePaths3 = sorted(list(glob.glob(basePath3)))
        for housePath in housePaths3:
            image0 = cv2.imread(housePaths0[0])
            image1 = cv2.imread(housePaths1[0])
            image2 = cv2.imread(housePaths2[0])
            image3 = cv2.imread(housePaths3[0])
            image0 = cv2.cvtColor(image0, cv2.COLOR_BGR2RGB)
            image1 = cv2.cvtColor(image1, cv2.COLOR_BGR2RGB)
            image2 = cv2.cvtColor(image2, cv2.COLOR_BGR2RGB)
            image3 = cv2.cvtColor(image3, cv2.COLOR_BGR2RGB)
            image0 = cv2.resize(image0, (size, size))
            image1 = cv2.resize(image1, (size, size))
            image2 = cv2.resize(image2, (size, size))
            image3 = cv2.resize(image3, (size, size))
            image_h0 = cv2.hconcat([image0, image1])
            image_h1 = cv2.hconcat([image2, image3])
            image = cv2.vconcat([image_h0, image_h1])
        images.append(image)
    return np.array(images) / 255.0

# load train images
inputPath = '/kaggle/input/5th-datarobot-ai-academy-deep-learning/images/train_images/'
inputPath2 = '/kaggle/input/5th-datarobot-ai-academy-deep-learning/images/test_images/'
size = 32
train_images = load_images_unit(train,inputPath,size)
test_images = load_images_unit(test,inputPath2,size)
#display(train_images.shape)
#display(train_images[0][0][0])
#画像をそれぞれ読み込み
def load_images(df,inputPath,size,roomType):
    images = []
    for i in df['id']:
        basePath = os.path.sep.join([inputPath, "{}_{}*".format(i,roomType)])
        housePaths = sorted(list(glob.glob(basePath)))
        for housePath in housePaths:
            image = cv2.imread(housePath)
            image = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
            image = cv2.resize(image, (size, size))
        images.append(image)
    return np.array(images) / 255.0

# load train images
inputPath = '/kaggle/input/5th-datarobot-ai-academy-deep-learning/images/train_images/'
inputPath2 = '/kaggle/input/5th-datarobot-ai-academy-deep-learning/images/test_images/'
size = 64
roomType = 'bathroom'
train_bathroom = load_images(train,inputPath,size,roomType)
test_bathroom = load_images(test,inputPath2,size,roomType)
roomType = 'bedroom'
train_bedroom = load_images(train,inputPath,size,roomType)
test_bedroom = load_images(test,inputPath2,size,roomType)
roomType = 'frontal'
train_frontal = load_images(train,inputPath,size,roomType)
test_frontal = load_images(test,inputPath2,size,roomType)
roomType = 'kitchen'
train_kitchen = load_images(train,inputPath,size,roomType)
test_kitchen = load_images(test,inputPath2,size,roomType)
#display(train_images.shape)
#display(train_images[0][0][0])
def create_model(inputShape ,TableShape):    
    
    backbone = VGG16(weights='imagenet',
                     include_top=False,
                     input_shape=inputShape)
    
    for layer in backbone.layers[:15]:
        layer.trainable = False

    vgg = Sequential(layers=backbone.layers)   
    
    vgg.add(GlobalAveragePooling2D())
    vgg.add(Dense(units=256, activation='relu',kernel_initializer='he_normal')) 
    vgg.add(Dropout(0.2))
    vgg.add(Dense(units=32, activation='relu',kernel_initializer='he_normal'))    
    vgg.add(Dense(units=1, activation='linear'))    
    

    L1_L2 = regularizers.l1_l2(l1=0.005,l2=0.005)
    input_df=Input(shape=(len(TableShape),))
    
    nn = Sequential()
    
    nn = Dense(units=256,input_shape=(TableShape,),kernel_initializer='he_normal',activation='relu')(input_df)
    nn = BatchNormalization()(nn)
    
    nn = Dense(units=128,kernel_initializer='he_normal',activation='relu')(nn)
    nn = Dropout(0.2)(nn)
    
    nn = Dense(units=64,kernel_initializer='he_normal',activation='relu')(nn)
    nn = Dropout(0.2)(nn)
    
    nn = Dense(units=32,kernel_initializer='he_normal',activation='relu',kernel_regularizer=L1_L2)(nn)
    nn = Dropout(0.2)(nn)
    
    nn = Model(inputs=input_df, outputs=nn)


    merge = concatenate([vgg.output, nn.output])
#     merge = concatenate([vgg.output, cnn.output, nn.output])
    
    mm = Dense(units=256, activation='relu',kernel_initializer='he_normal')(merge)
    mm = Dropout(0.2)(mm)
    mm = Dense(units=32, activation='relu',kernel_initializer='he_normal',kernel_regularizer=L1_L2)(mm)
    mm = Dropout(0.2)(mm)
    mm = Dense(units=1, activation='linear')(mm)
    model = Model(inputs=[vgg.input,nn.input],outputs=mm) 
    model.compile(loss='mape', optimizer='adam', metrics=['mape'])
    
    return model
train_x = train.drop(['price', 'id'], axis=1)
train_y = train.price
test_x = test.drop(['id'], axis=1)
def mean_absolute_percentage_error(y_true, y_pred): 
    y_true, y_pred = np.array(y_true), np.array(y_pred)
    return np.mean(np.abs((y_true - y_pred) / y_true)) * 100
#CV
scores = []

kf = KFold(n_splits=3, shuffle=True)

for i, (train_ix, valid_ix) in enumerate(kf.split(train_x)):

    train_x_, train_y_ = train_x.iloc[train_ix].values, train_y.iloc[train_ix].values
    valid_x_, valid_y_ = train_x.iloc[valid_ix].values, train_y.iloc[valid_ix].values
    
    train_image_x_ = train_images[train_ix]
    valid_image_x_ = train_images[valid_ix]
    
    # callback parameter
    filepath = "cnn_best_model_CV"+str(i)+".hdf5" 
    es = EarlyStopping(patience=5, mode='min', verbose=1) 
    checkpoint = ModelCheckpoint(monitor='val_loss', filepath=filepath, save_best_only=True, mode='auto') 
    reduce_lr_loss = ReduceLROnPlateau(monitor='val_loss',  patience=5, verbose=1,  mode='min')

    # 訓練実行
    inputShape = (size, size, 3)
    tableShape = cols
    
    model = create_model(inputShape,tableShape)
    history = model.fit([train_image_x_, train_x_],train_y_, 
                    validation_data=([valid_image_x_, valid_x_],valid_y_),
                    epochs=100, batch_size=32,callbacks=[es, checkpoint, reduce_lr_loss])
    
    # load best model weights
    if os.path.exists(filepath):
        model.load_weights(filepath)

    # 評価
    valid_pred = model.predict([valid_image_x_, valid_x_],batch_size=5).reshape((-1,1))
    mape_score = mean_absolute_percentage_error(valid_y_, valid_pred)
loss = history.history['loss']
val_loss = history.history['val_loss']
epochs = range(len(loss))
plt.plot(epochs, loss, 'bo' ,label = 'training loss')
plt.plot(epochs, val_loss, 'b' , label= 'validation loss')
plt.title('Training and Validation loss')
plt.legend()
plt.show()
plot_model(model, to_file='cnn.png')
model.summary()
print (mape_score)
price1 = model.predict([test_images,test_x], batch_size=16).reshape((-1,1))
#price1
submission.price = price1 
submission.to_csv('./submission.csv')
submission.head()