import pandas as pd
import numpy as np
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.pipeline import Pipeline
from sklearn.model_selection import train_test_split
from sklearn.ensemble import VotingClassifier
from sklearn.naive_bayes import MultinomialNB
from sklearn.linear_model import SGDClassifier
from sklearn.ensemble import RandomForestClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.preprocessing import LabelEncoder
from keras.utils import np_utils
from keras.preprocessing.image import ImageDataGenerator
from keras.models import Sequential
from keras.layers import Conv2D, MaxPooling2D
from keras.layers import Activation, Dropout, Flatten, Dense
from keras.layers import ZeroPadding2D
from keras.layers import Convolution2D
import h5py
from keras import applications
from keras.utils.np_utils import to_categorical
from keras.models import Model
import urllib
import os
import math

"""
Explanation: Stackline - DS Assignment
Import functions
End of explanation
"""


df_train = pd.read_excel('Training_Data_Assessment.xlsx')
df_test = pd.read_excel('Data To Classify_Assessment.xlsx')
categories = pd.read_excel('Categories_Assessment.xlsx',header=None,names=['CategoryName'])

df_train.head()

df_train.describe()

df_test.head()

df_test.describe()

categories.head()

unique_categories = categories['CategoryName']

"""
Explanation: Exploratory Data Analysis
Reading Data
End of explanation
"""


df_train.drop_duplicates('ASIN',inplace=True)

"""
Explanation: Observations:


Training data: 6031 unique ASIN in 6034 rows. Need to remove duplicates.


Testing data: 57030 unique ASIN in 57030 rows. No duplicates found.


No Null values observed


Removing duplicate ASINs
End of explanation
"""


x_df_train, x_df_test, y_df_train, y_df_test = train_test_split(df_train[['Title','ImageUrl']],df_train[['CategoryName']],
train_size=.8, stratify=df_train['CategoryName'],random_state=30)

"""
Explanation: Training and Validation Split
End of explanation
"""


current_path = os.getcwd()
train_path = os.path.join(current_path,'train')
validation_path = os.path.join(current_path,'validation')
test_path = os.path.join(current_path,'test')

"""
Explanation: Image Downloading
Aim

To download training, validation and testing images from image url
The images are stored as
        /train
            /category1
                train_image1.jpg
                train_image2.jpg
            /category2
                train_image1.jpg
                train_image2.jpg
        /validation
            /category1
                validation_image1.jpg
                validation_image2.jpg
            /category2
                validation_image1.jpg
                validation_image2.jpg
        /test
            /New folder
                test_image1.jpg
                test_image2.jpg

Path variables
End of explanation
"""


def image_download(x_df,y_df,unique_categories,path,category_column='CategoryName',imageurl_column='ImageUrl'):
    for i in x_df.index:
        for j in unique_categories:            
            if y_df[category_column][i] == j:
                class_path = os.path.join(path,j)
                if not os.path.exists(class_path):
                    os.makedirs(class_path)
                filename = '%s.jpg'%i
                fullfilename = os.path.join(class_path,filename)
                if not os.path.exists(fullfilename):
                    try:
                        urllib.request.urlretrieve(x_df[imageurl_column][i],fullfilename)
                    except:
                        print('Did not download:')
                        print(i)
                        print(x_df[imageurl_column][i])
                        pass
    return

"""
Explanation: Function to download test and validation images
End of explanation
"""


image_download(x_df=x_df_train,y_df=y_df_train,unique_categories=unique_categories,path=train_path)

"""
Explanation: Download Training Images
End of explanation
"""


image_download(x_df=x_df_test,y_df=y_df_test,unique_categories=unique_categories,path=validation_path)

"""
Explanation: Download Validation Images
End of explanation
"""


for i in df_test.index:
    class_path = os.path.join(test_path,'New folder')
    if not os.path.exists(class_path):
        os.makedirs(class_path)
    filename = '%s.jpg'%i
    fullfilename = os.path.join(class_path,filename)
    if not os.path.exists(fullfilename):
        try:
            urllib.request.urlretrieve(df_test['ImageUrl'][i],fullfilename)
        except:
            print('Did not download:')
            print(i)
            print(df_test['ImageUrl'][i])
            pass

"""
Explanation: Download Test Images
End of explanation
"""


text_clf_multinomial_nb = Pipeline([('vect', CountVectorizer(stop_words='english',lowercase=True)),('tfidf', TfidfTransformer()),('clf', MultinomialNB())])
text_clf_multinomial_nb.fit(x_df_train['Title'], y_df_train['CategoryName'])
text_clf_multinomial_nb.score(x_df_test['Title'],y_df_test['CategoryName'])

"""
Explanation: Text Classification
Aim

To build a pipeline for text features
To build classifiers for the text features

Steps in pipeline

Convert to lowercase
Remove stop words
Convert a collection of text documents to a matrix of token counts
Transform a count matrix to a normalized tf or tf-idf representation. Tf means term-frequency while tf-idf means term-frequency times inverse document-frequency. This is a common term weighting scheme in information retrieval, that has also found good use in document classification.

Classifiers used

Multinomial Naive Bayes
Linear SVM with SGD
Random Forests
Logistic Regression

Multinomial NB
End of explanation
"""


text_clf_sgd = Pipeline([('vect', CountVectorizer(stop_words='english',lowercase=True)),('tfidf', TfidfTransformer()),('clf', SGDClassifier())])
text_clf_sgd.fit(x_df_train['Title'], y_df_train['CategoryName'])
text_clf_sgd.score(x_df_test['Title'],y_df_test['CategoryName'])

"""
Explanation: SGD Classifier
End of explanation
"""


text_clf_rf = Pipeline([('vect', CountVectorizer(stop_words='english',lowercase=True)),('tfidf', TfidfTransformer()),('clf', RandomForestClassifier())])
text_clf_rf.fit(x_df_train['Title'], y_df_train['CategoryName'])
text_clf_rf.score(x_df_test['Title'],y_df_test['CategoryName'])

"""
Explanation: Random Forest Classifier
End of explanation
"""


text_clf_logis = Pipeline([('vect', CountVectorizer(stop_words='english',lowercase=True)),('tfidf', TfidfTransformer()),('clf', LogisticRegression())])
text_clf_logis.fit(x_df_train['Title'], y_df_train['CategoryName'])
text_clf_logis.score(x_df_test['Title'],y_df_test['CategoryName'])

"""
Explanation: Logistic Regression Classifier
End of explanation
"""


# dimensions of our images.
img_width, img_height = 150, 150

top_model_weights_path = 'bottleneck_fc_model1.h5'
train_data_dir = 'train'
validation_data_dir = 'validation'
epochs = 50
batch_size = 32

datagen = ImageDataGenerator(rescale=1. / 255)

"""
Explanation: Note

Due to time and computational constraints, I am not going be doing a grid search cross validation for the above models. 
I believe that a test dataset accuracy of above 80% is sufficient enough as I am going to take the average of all classifiers.

Image Classification
Aim

To train a classifier over a pre-trained VGG16 model

Define Variables
End of explanation
"""


model = applications.VGG16(include_top=False, weights='imagenet')

"""
Explanation: Building VGG16 network
End of explanation
"""


#train generator
train_generator = datagen.flow_from_directory(
    train_data_dir,
    target_size=(img_width, img_height),
    batch_size=batch_size,
    class_mode= 'categorical',
    shuffle=False)

#train variables
nb_train_samples = len(train_generator.filenames)
num_classes = len(train_generator.class_indices)
predict_size_train = int(math.ceil(nb_train_samples / batch_size))

#creaing training feature set
bottleneck_features_train = model.predict_generator(
    train_generator, predict_size_train)
np.save('bottleneck_features_train',bottleneck_features_train)

#train labels
train_labels = train_generator.classes
train_labels = to_categorical(train_labels,63)

"""
Explanation: Running VGG16 is expensive, as working on CPU. I want to only do it only once. This prevents me from using data augmentation which can improve my model by preventing overfitting.
Generating Training Feature Set
End of explanation
"""


#validation generator
generator = datagen.flow_from_directory(
    validation_data_dir,
    target_size=(img_width, img_height),
    batch_size=batch_size,
    class_mode= 'categorical',
    shuffle=False)

#test variables
nb_validation_samples = len(generator.filenames)
predict_size_validation = int(math.ceil(nb_validation_samples / batch_size))

#creaing validation feature set
bottleneck_features_validation = model.predict_generator(
    generator, predict_size_validation)
np.save('bottleneck_features_validation',bottleneck_features_validation)

#test labels
validation_labels = generator.classes
validation_labels = to_categorical(validation_labels,63)

"""
Explanation: Generating Validation Feature Set
End of explanation
"""


#loading training and validation feature set generated above
train_data = np.load('bottleneck_features_train.npy')
validation_data = np.load('bottleneck_features_validation.npy')

"""
Explanation: Loading Training and Validation feature set
End of explanation
"""


from keras import optimizers
rmsprop = optimizers.RMSprop(lr=0.001)

top_model = Sequential()
top_model.add(Flatten(input_shape=train_data.shape[1:]))
top_model.add(Dense(256, activation='relu'))
top_model.add(Dropout(0.6))
top_model.add(Dense(63, activation='softmax'))

top_model.compile(optimizer=rmsprop,loss='categorical_crossentropy', metrics=['accuracy'])

top_model.fit(train_data, train_labels,
        epochs=30,
        batch_size=batch_size,
        validation_data=(validation_data, validation_labels))
top_model.save_weights(top_model_weights_path)

"""
Explanation: Training top classifier model
End of explanation
"""


model_vgg = applications.VGG16(weights='imagenet', include_top=False, input_shape=(150, 150, 3))

top_model = Sequential()
top_model.add(Flatten(input_shape=model_vgg.output_shape[1:]))
top_model.add(Dense(256, activation='relu'))
top_model.add(Dropout(0.6))
top_model.add(Dense(63, activation='softmax'))

top_model.load_weights('bottleneck_fc_model1.h5')

model = Model(inputs = model_vgg.input, outputs = top_model(model_vgg.output))

"""
Explanation: Making Final Image Recognition Model
End of explanation
"""


df_test.reset_index(inplace=True)
filename=[]
vgg16_predictions=[]

final_data_dir='test'
test_datagen = ImageDataGenerator(rescale=1./255)
test_generator = test_datagen.flow_from_directory(
        final_data_dir,
        target_size=(img_width, img_height),
        class_mode=None,
        batch_size=1)

filenames = test_generator.filenames
nb_samples = len(filenames)

predict = model.predict_generator(test_generator,steps = nb_samples)

"""
Explanation: Predict Category based on Test Images
End of explanation
"""


for f in filenames:
    filename.append(f)
for r in predict:
    vgg16_predictions.append(np.argmax(r))

ir_res = pd.DataFrame(
    {'filename': [f[11:-4] for f in filename],
     'category': vgg16_predictions,
    })
ir_res.to_csv('ir_res.csv')
len(ir_res)

"""
Explanation: Make image classification results into a data frame
End of explanation
"""


x=pd.read_csv('ir_res.csv')
index=train_generator.class_indices
y=list(x['category'])
categories_p=[]
for i in y:
    for j in index:
        if i == index[j]:
            categories_p.append(j)

"""
Explanation: Final Product Classification
Getting category name from index
End of explanation
"""


df_test.set_index('ASIN',inplace=True)

a=text_clf_multinomial_nb.predict(df_test['Title'])
b=text_clf_sgd.predict(df_test['Title'])
c=text_clf_rf.predict(df_test['Title'])
d=text_clf_logis.predict(df_test['Title'])

ir_res_new = pd.DataFrame(
    {'ASIN': x['filename'],
     'CNN Category': categories_p,
    })
ir_res_new.set_index('ASIN',inplace=True)

predictions = df_test.join(ir_res_new)
predictions.drop(['Title','ImageUrl','BrandName'],1, inplace=True)
predictions['Multinomial NB Category'] = a
predictions['SVC Category'] = b
predictions['RF Category'] = c
predictions['LogisticR Category'] = d

"""
Explanation: Collecting all predictions from all classifiers
End of explanation
"""


final_pred =[]
for i in range(len(df_test)):
    lst = [a[i],b[i],c[i],d[i],predictions['CNN Category'][i]]
    final_pred.append(max(set(lst), key=lst.count))
df_test['Category'] = final_pred

df_test.to_csv('final_prediction.csv')
predictions.to_csv('predictions_allClassifiers.csv')

"""
Explanation: Take Average of all 5 classifiers
End of explanation
"""


