import datapot as dp
from datapot import datasets

import pandas as pd
from __future__ import print_function
import sys
import bz2
import time
import xgboost as xgb
from sklearn.model_selection import cross_val_score

"""
Explanation: DataPot Usage Examples
End of explanation
"""


data_imdb = datasets.load_imdb() 

datapot_imdb = dp.DataPot()

t0 = time.time()
datapot_imdb.detect(data_imdb)
print('detect time:', time.time()-t0)
datapot_imdb

datapot_imdb.remove_transformer('sentiment', 0)

t0 = time.time()
datapot_imdb.fit(data_imdb, verbose=True)

print('fit time:', time.time()-t0)

t0 = time.time()
df_imdb = datapot_imdb.transform(data_imdb)
print('transform time:', time.time()-t0)

df_imdb.head()

X = df_imdb.drop(['sentiment'], axis=1)
y = df_imdb['sentiment']

model = xgb.XGBClassifier()
cv_score = cross_val_score(model, X, y, cv=5)
assert all(i > 0.5 for i in cv_score), 'Low score!'
print('Cross-val score:', cv_score)

model.fit(X, y)
fi = model.feature_importances_

print('Feature importance:')
print(*(list(zip(X.columns, fi))), sep='\n')

"""
Explanation: Bag of Words Meets Bags of Popcorn
Usage example for unstructured textual bzip2-compressed data
datapot.fit method subsamples the data to detect language and choose corresponding stopwords and stemming.
For each review datapot.transform generates an SVD-compressed 12-dimensional tfidf-vector representation.
End of explanation
"""


data_job = datasets.load_job_salary()

datapot_job = dp.DataPot()

t0 = time.time()
datapot_job.detect(data_job)
print('detect time:', time.time()-t0)
datapot_job

t0 = time.time()
datapot_job.fit(data_job, verbose=True)
print('fit time:', time.time()-t0)

t0 = time.time()
df_job = datapot_job.transform(data_job)
print('transform time:', time.time()-t0)

print(df_job.columns)
print(df_job.shape)
df_job.head()

X_job = df_job.drop(['SalaryNormalized', 'Id'], axis=1)
y_job = pd.qcut(df_job['SalaryNormalized'].values, q=2, labels=[0,1]).ravel()

model = xgb.XGBClassifier()
cv_score_job = cross_val_score(model, X_job, y_job, cv=5)
print('Cross-val score:', cv_score_job)
assert all(i > 0.5 for i in cv_score_job), 'Low score!'

model.fit(X_job, y_job)
fi_job = model.feature_importances_

print('Feature importance:')
print(*(list(zip(X_job.columns, fi_job))), sep='\n')


"""
Explanation: Job Salary Prediction
Usage example for unstructured textual bzip2-compressed data
End of explanation
"""


