import sys
import pandas as pd
from sqlalchemy import create_engine
import nltk
nltk.download(['punkt', 'wordnet', 'averaged_perceptron_tagger'])
import re
from nltk.tokenize import word_tokenize
from nltk.stem import WordNetLemmatizer
from sklearn.metrics import classification_report
from sklearn.metrics import confusion_matrix
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
from sklearn.pipeline import Pipeline, FeatureUnion
from sklearn.base import BaseEstimator, TransformerMixin
from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
import pickle
from sklearn.model_selection import GridSearchCV

def load_data(database_filepath):
    """
    load data from sql db
    :param database_filepath: sql db path
    :return: pandas dataframe
    """
    engine = create_engine("sqlite:///"+database_filepath)
    df = pd.read_sql_table('modeling_data', engine)
    yvar = [item for item in list(df) if item not in ['message', 'original', 'genre', 'id']]
    X = df['message']
    Y = df[yvar]
    return X.values, Y.values, list(Y)


def tokenize(text):
    """
    processing the text input
    :param text: text inputs
    :return:
    """
    url_regex = 'http[s]?://(?:[a-zA-Z]|[0-9]|[$-_@.&+]|[!*\(\),]|(?:%[0-9a-fA-F][0-9a-fA-F]))+'
    detected_urls = re.findall(url_regex, text)
    for url in detected_urls:
        text = text.replace(url, "urlplaceholder")

    tokens = word_tokenize(text)
    lemmatizer = WordNetLemmatizer()

    clean_tokens = []
    for tok in tokens:
        clean_tok = lemmatizer.lemmatize(tok).lower().strip()
        clean_tokens.append(clean_tok)

    return clean_tokens


def build_model():
    """
    build model pipeline
    :return: model pipeline
    """
    model_pipeline = Pipeline([
        ('features', Pipeline([
            ('vect', CountVectorizer(tokenizer=tokenize)),
            ('tfidf', TfidfTransformer())
        ])),
        ('clf', RandomForestClassifier())
    ])
    return model_pipeline

def model_gridsearch(model, parameters):
    cv = GridSearchCV(model, param_grid=parameters, verbose=3)
    return cv

def evaluate_model(model, X_test, Y_test, category_names):
    """
    evaluate model performances
    :param model:  model obj
    :param X_test:  test x
    :param Y_test:  test y
    :param category_names:  y names
    :return:
    """
    y_pred = model.predict(X_test)
    print(classification_report(Y_test, y_pred, target_names=category_names))


def save_model(model, model_filepath):
    """
    save model to local path
    :param model:  model obj
    :param model_filepath:  saving path
    :return:
    """
    with open(model_filepath, 'wb') as f:
        pickle.dump(model, f)


def main():
    """
    CLI to fit the model
    :return:
    """

    if len(sys.argv) == 3:
        database_filepath, model_filepath = sys.argv[1:]
        print('Loading data...\n    DATABASE: {}'.format(database_filepath))
        X, Y, category_names = load_data(database_filepath)
        X_train, X_test, Y_train, Y_test = train_test_split(X, Y, test_size=0.2)
        
        print('Building model...')
        model = build_model()
        
        print('Training model...')
        # model.fit(X_train, Y_train)

        parameters = {
            'clf__n_estimators': [100, 400, 800],
            #     'clf__criterion':["gini", "entropy"]
        }
        cv = model_gridsearch(model, parameters)
        best_model_pipeline = cv.best_estimator_

        print('Evaluating model...')
        evaluate_model(best_model_pipeline, X_test, Y_test, category_names)

        print('Saving model...\n    MODEL: {}'.format(model_filepath))
        save_model(best_model_pipeline, model_filepath)

        print('Trained model saved!')

    else:
        print('Please provide the filepath of the disaster messages database '\
              'as the first argument and the filepath of the pickle file to '\
              'save the model to as the second argument. \n\nExample: python '\
              'train_classifier.py ../data/DisasterResponse.db classifier.pkl')


if __name__ == '__main__':
    main()
[(5, 'nltk.download', 'nltk.download', False, 'import nltk\n'), (25, 'sqlalchemy.create_engine', 'create_engine', False, 'from sqlalchemy import create_engine\n'), (26, 'pandas.read_sql_table', 'pd.read_sql_table', True, 'import pandas as pd\n'), (40, 're.findall', 're.findall', False, 'import re\n'), (44, 'nltk.tokenize.word_tokenize', 'word_tokenize', False, 'from nltk.tokenize import word_tokenize\n'), (45, 'nltk.stem.WordNetLemmatizer', 'WordNetLemmatizer', False, 'from nltk.stem import WordNetLemmatizer\n'), (70, 'sklearn.model_selection.GridSearchCV', 'GridSearchCV', False, 'from sklearn.model_selection import GridSearchCV\n'), (83, 'sklearn.metrics.classification_report', 'classification_report', False, 'from sklearn.metrics import classification_report\n'), (94, 'pickle.dump', 'pickle.dump', False, 'import pickle\n'), (107, 'sklearn.model_selection.train_test_split', 'train_test_split', False, 'from sklearn.model_selection import train_test_split\n'), (65, 'sklearn.ensemble.RandomForestClassifier', 'RandomForestClassifier', False, 'from sklearn.ensemble import RandomForestClassifier\n'), (62, 'sklearn.feature_extraction.text.CountVectorizer', 'CountVectorizer', False, 'from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer\n'), (63, 'sklearn.feature_extraction.text.TfidfTransformer', 'TfidfTransformer', False, 'from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer\n')]