#! tar -xf data/aclImdb.tar.bz2 --directory data

from sklearn.datasets import load_files

reviews_train = load_files("data/aclImdb/train/")
text_train, y_train = reviews_train.data, reviews_train.target

print("Number of documents in training data: %d" % len(text_train))
print(np.bincount(y_train))

reviews_test = load_files("data/aclImdb/test/")
text_test, y_test = reviews_test.data, reviews_test.target
print("Number of documents in test data: %d" % len(text_test))
print(np.bincount(y_test))

print(text_train[1])

print(y_train[1])

from sklearn.feature_extraction.text import CountVectorizer
cv = CountVectorizer()
cv.fit(text_train)

len(cv.vocabulary_)

print(cv.get_feature_names()[:50])
print(cv.get_feature_names()[50000:50050])

X_train = cv.transform(text_train)
X_train

print(text_train[19726])

X_train[19726].nonzero()[1]

X_test = cv.transform(text_test)

from sklearn.svm import LinearSVC

svm = LinearSVC()
svm.fit(X_train, y_train)

svm.score(X_train, y_train)

svm.score(X_test, y_test)

def visualize_coefficients(classifier, feature_names, n_top_features=25):
    # get coefficients with large absolute values 
    coef = classifier.coef_.ravel()
    positive_coefficients = np.argsort(coef)[-n_top_features:]
    negative_coefficients = np.argsort(coef)[:n_top_features]
    interesting_coefficients = np.hstack([negative_coefficients, positive_coefficients])
    # plot them
    plt.figure(figsize=(15, 5))
    colors = ["red" if c < 0 else "blue" for c in coef[interesting_coefficients]]
    plt.bar(np.arange(2 * n_top_features), coef[interesting_coefficients], color=colors)
    feature_names = np.array(feature_names)
    plt.subplots_adjust(bottom=0.3)
    plt.xticks(np.arange(1, 1 + 2 * n_top_features), feature_names[interesting_coefficients], rotation=60, ha="right");


visualize_coefficients(svm, cv.get_feature_names())

svm = LinearSVC(C=0.001)
svm.fit(X_train, y_train)
svm.score(X_test, y_test)

visualize_coefficients(svm, cv.get_feature_names())

from sklearn.pipeline import make_pipeline
text_pipe = make_pipeline(CountVectorizer(), LinearSVC())
text_pipe.fit(text_train, y_train)
text_pipe.score(text_test, y_test)

from sklearn.grid_search import GridSearchCV

param_grid = {'linearsvc__C': np.logspace(-5, 0, 6)}
grid = GridSearchCV(text_pipe, param_grid, cv=5)
grid.fit(text_train, y_train)

grid.best_params_

visualize_coefficients(grid.best_estimator_.named_steps['linearsvc'],
                       grid.best_estimator_.named_steps['countvectorizer'].get_feature_names())

grid.score(text_test, y_test)

"""
Explanation: Text Classification of Movie Reviews
Unpack data - this only works on linux and (maybe?) OS X. Unpack using 7zip on Windows.
End of explanation
"""


text_pipe = make_pipeline(CountVectorizer(), LinearSVC())

param_grid = {'linearsvc__C': np.logspace(-3, 2, 6),
              "countvectorizer__ngram_range": [(1, 1), (1, 2)]}

grid = GridSearchCV(text_pipe, param_grid, cv=5)

grid.fit(text_train, y_train)

scores = np.array([score.mean_validation_score for score in grid.grid_scores_]).reshape(3, -1)
plt.matshow(scores)
plt.ylabel("n-gram range")
plt.yticks(range(3), param_grid["countvectorizer__ngram_range"])
plt.xlabel("C")
plt.xticks(range(6), param_grid["linearsvc__C"]);
plt.colorbar()

grid.best_params_

visualize_coefficients(grid.best_estimator_.named_steps['linearsvc'],
                       grid.best_estimator_.named_steps['countvectorizer'].get_feature_names())

grid.score(text_test, y_test)

"""
Explanation: N-Grams
End of explanation
"""


