def prepare_data(df):
    y = df.category
    X = df[['title', 'additional_info']].values 
    print("X:", X.shape, " y:", y.shape)
    return X,y

X, y = prepare_data(g)

def join_columns(X):
    X_joined = []
    for x in X:
        X_joined.append(str(x[0]) + str(" - ") + str(x[1]))
    return np.array(X_joined).reshape(-1, 1)

def correct_type(X):
    X_corrected = []
    for x in X:
        if x == np.nan:
            X_corrected.append("None")
        else:
            X_corrected.append(str(x))
    return np.array(X_corrected).reshape(-1, 1)

X_title = correct_type(X[:,0])
X_additional = correct_type(X[:,1])
X_joined = join_columns(X)
print(X_joined.shape, X_title.shape, X_additional.shape)

X_joined[4][0]

X_title[4][0]

X_additional[4][0]

"""
Explanation: Preparing Dataset
End of explanation
"""


X_train, X_test, y_train, y_test = train_test_split(X_joined, y, test_size=0.3, stratify=y)
X_train2, X_test2, y_train2, y_test2 = train_test_split(X_title, y, test_size=0.3, stratify=y)
X_train3, X_test3, y_train3, y_test3 = train_test_split(X_additional, y, test_size=0.3, stratify=y)

print(X_train.shape, X_test.shape, y_train.shape, y_test.shape)

"""
Explanation: Split dataset into train and test
I have three sets of data:
+ Set 1: combination of title and additional information
+ Set 2: title
+ Set 3: additional information
End of explanation
"""


# Spacy for tokenization and lemmatization
en_nlp = spacy.load('en')

# List of NLTK stemmers to the tested 
snowball = nltk.stem.SnowballStemmer("english")
lancaster = nltk.stem.LancasterStemmer()
porter = nltk.stem.PorterStemmer()

def compare_normalization(doc):
    # tokenize document in spacy
    doc_spacy = en_nlp(doc)
    # Lemmatization
    print("Lemmatization:")
    print([token.lemma_ for token in doc_spacy])
    # print tokens found by Porter stemmer
    print("Porter:")
    print([porter.stem(token.norm_.lower()) for token in doc_spacy])
    print("snowball:")
    print([snowball.stem(token.norm_.lower()) for token in doc_spacy])    
    print("lancaster:")
    print([lancaster.stem(token.norm_.lower()) for token in doc_spacy])    

compare_normalization(u"this is an informal reception at our apartment complex before we have a wedding. we would like to spend the first 1-2 hours on portraits of just the two of us at nice spots around our apartment complex or any other ideas you have. we would like some nice portraits. then capture a bit of the event.")

compare_normalization(u"i would like to have the photographer take a lot of pics of the president (rakesh kumar) mingling with the guests. also, individualized couples and family pics by the fireplace. i would like photos of decorations, desserts, displays etc. i can give more direction once i meet with the photographer.")

"""
Explanation: Normalization: Stemming vs Lemmatization
End of explanation
"""


import re
regexp = re.compile('(?u)\\b\\w\\w+\\b') # regexp used in CountVectorizer

old_tokenizer = en_nlp.tokenizer
# replacing the tokenizer
en_nlp.tokenizer = lambda string: old_tokenizer.tokens_from_list(regexp.findall(string))

def custom_tokenizer(document):
    doc_spacy = en_nlp(document, entity=False, parse=False) 
    return [token.lemma_ for token in doc_spacy]

"""
Explanation: Lemmatization seem to mantain keywords in context. I will use lemmatization on the vectorizers
End of explanation
"""


from sklearn.feature_extraction import text 

my_additional_stop_words = ["nan", "test", "fdghjkl", "test2", "dddfgb2v", "etc"]

s_words = text.ENGLISH_STOP_WORDS.union(my_additional_stop_words)

"""
Explanation: Stop words
I used the stop words during the cleanup process
End of explanation
"""


count_vect = CountVectorizer(analyzer=u'word', tokenizer=custom_tokenizer, stop_words=s_words, ngram_range=(1,2), min_df=3)

X_train_counts = count_vect.fit_transform(X_train.ravel())
X_test_counts = count_vect.transform(X_test.ravel())
print(X_train_counts.shape, X_test_counts.shape, "Vocabulary size: ", len(count_vect.vocabulary_))

"""
Explanation: Vectorizers
Count Vectorizer
End of explanation
"""


count_vect_bin = CountVectorizer(analyzer=u'word', tokenizer=custom_tokenizer, binary=True, stop_words=s_words, ngram_range=(1,2), min_df=3)

X_train_counts_bin = count_vect_bin.fit_transform(X_train.ravel())
X_test_counts_bin = count_vect_bin.transform(X_test.ravel())
print(X_train_counts_bin.shape, X_test_counts_bin.shape, "Vocabulary size: ", len(count_vect_bin.vocabulary_))

"""
Explanation: Binary Count Vectorizer
End of explanation
"""


tf = TfidfTransformer(use_idf=False)
X_train_tf = tf.fit_transform(X_train_counts)
X_test_tf = tf.transform(X_test_counts)
print(X_train_tf.shape, X_test_tf.shape)

"""
Explanation: Term Frequency Vectorizer
End of explanation
"""


tf_idf = TfidfVectorizer(tokenizer=custom_tokenizer, stop_words=s_words, ngram_range=(1,2), min_df=3)

X_train_tfidf = tf_idf.fit_transform(X_train.ravel())
X_test_tfidf = tf_idf.transform(X_test.ravel())
print(X_train_tfidf.shape, X_test_tfidf.shape, "Vocabulary size: ", len(tf_idf.vocabulary_))

"""
Explanation: Tf-idf Vectorizer
An even more effective way to vectorize text data is by using Tfidf. In brief, you can multiply the term frequency of the words that compose a document by the inverse document frequency of the word itself (that is, in the number of documents it appears, if logarithmically scaled). This is very handy to highlight words that effectively describe each document, and is a powerful discriminative element among the dataset.
End of explanation
"""


#Baseline models:
models = []
models.append(('MNB', MultinomialNB()))
models.append(('CART', DecisionTreeClassifier())) 
models.append(('KNN', KNeighborsClassifier())) 
models.append(('LR', LogisticRegression(multi_class='multinomial', solver='newton-cg'))) 
models.append(('SVM', LinearSVC())) 

# Test options and evaluation metric
num_folds = 5
seed = 7
scoring = 'accuracy'

def compute_baseline(models, X_train_counts, y_train, num_folds, scoring):
    results = []
    names = []
    for name, model in models:
        kfold = KFold(n_splits=num_folds, random_state=seed)
        cv_results = cross_val_score(model, X_train_counts, y_train, cv=kfold, scoring=scoring)
        results.append(cv_results)
        names.append(name)
        print("[{}] \t{:.3f} ({:.3f})".format(name, cv_results.mean(), cv_results.std()))

    return names, results

def plot_baseline(names, results, classes=9, order=True, guess=True):
    #sorting prior to plotting
    if order == True:
        rm = np.median(results, axis=1)
        idxs = np.argsort(rm)[::-1]
        names = np.array(names)[idxs]
        r = []
        for i in idxs:
            r.append(results[i])
        results = r   
    fig = plt.figure() 
    plt.title('Algorithm Comparison')
    plt.ylabel("Accuracy Score")
    ax = fig.add_subplot(111) 
    plt.boxplot(results)
    if guess:
        plt.ylim(0,1)
        plt.hlines(y=1/classes, xmin=0, xmax =10, linestyles="--", label='Guessing', color='g')
        plt.legend()
    ax.set_xticklabels(names) 
    plt.show()

def evaluate_features(models, X_train_counts, y_train, num_folds, scoring, guess=True):
    names, results = compute_baseline(models, X_train_counts, y_train, num_folds, scoring)
    plot_baseline(names, results, order=True, classes=len(np.unique(y_train)), guess=guess)

"""
Explanation: Very short texts are likely to have noisy tf–idf values while the binary occurrence info is more stable.
Building Baseline model
According to Sklearn documentation:

Inherently multiclass: Naive Bayes, Decision Trees, Random Forests, Nearest Neighbors,  LogisticRegression ('multinomial')
End of explanation
"""


evaluate_features(models, X_train_counts, y_train, num_folds, scoring, guess=False)

"""
Explanation: Evaluating models with each of the different types of vectorizers created in the previous section
End of explanation
"""


evaluate_features(models, X_train_counts_bin, y_train, num_folds, scoring)

"""
Explanation: SVM and Logistic Regression do well with a simple count vertorizer
End of explanation
"""


evaluate_features(models, X_train_tf, y_train, num_folds, scoring)

evaluate_features(models, X_train_tfidf, y_train, num_folds, scoring)

"""
Explanation: TermFrequency
End of explanation
"""


# ensembles
ensembles = []
ensembles.append(('AB', AdaBoostClassifier())) 
ensembles.append(('RF', RandomForestClassifier())) 

evaluate_features(ensembles, X_train_counts_bin, y_train, num_folds, scoring, guess=True)

evaluate_features(ensembles, X_train_tf, y_train, num_folds, scoring, guess=True)

evaluate_features(ensembles, X_train_tfidf, y_train, num_folds, scoring, guess=True)

"""
Explanation: I will proceed to tune the parameters of SVM, LR and MNB

Emsemble methods

CART (Decision trees) seem to be performing OK. I am interested in seeing if Random Forest could boost that performance significantly.
End of explanation
"""


