import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer
from sklearn.metrics import accuracy_score, confusion_matrix
from sklearn.feature_extraction.text import TfidfTransformer

from sklearn.pipeline import Pipeline
from sklearn.metrics import classification_report
from sklearn.linear_model import SGDClassifier
from nltk import word_tokenize
import nltk
#nltk.download('punkt')
import re
import joblib

def train_intent():
     
    df = pd.read_csv("Training Data/intent_training_data.csv")
    df.head
    question = df["QUESTIONS"]
    intent = df["INTENT"]


    def preprocess(data):

        data = data.lower()
        # stop_words =['hers', 'between', 'yourself', 'but', 'again', 'there', 'about', 'once', 'during', 'out', 'very', 'having', 'with', 'they', 'own', 'an', 'be', 'some', 'for', 'do', 'its', 'yours', 'such', 'into', 'of', 'most', 'itself', 'other', 'off', 'is', 's', 'am', 'or', 'who', 'as', 'from', 'him', 'each', 'the', 'themselves', 'until', 'below', 'are', 'we', 'these', 'your', 'his', 'don', 'nor', 'me', 'were', 'her', 'more', 'himself', 'this', 'down', 'should', 'our', 'their', 'while', 'above', 'both', 'up', 'to', 'ours', 'had', 'she', 'all', 'no', 'when', 'at', 'any', 'before', 'them', 'same', 'and', 'been', 'have', 'in', 'will', 'on', 'does', 'yourselves', 'then', 'that', 'because', 'what', 'over', 'why', 'so', 'can', 'did', 'not', 'now', 'under', 'he', 'you', 'herself', 'has', 'just', 'where', 'too', 'only', 'myself', 'which', 'those', 'i', 'after', 'few', 'whom', 't', 'being', 'if', 'theirs', 'my', 'against', 'a', 'by', 'doing', 'it', 'how', 'further', 'was', 'here', 'than']
        # word_tokens = word_tokenize(data) 
        # data = [w for w in word_tokens if not w in stop_words] 
        # for w in word_tokens: 
        #     if w not in stop_words: 
        #         data.append(w) 
        # data = " ".join(data)
        data = re.sub(r'[^a-zA-Z0-9]', " ", data)
        return data

    question = question.apply(preprocess)
    X = question
    y = intent
    my_tags = list(set(intent))
    #print(my_tags)
    #X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state = 0)

    sgd = Pipeline([('vect', CountVectorizer()),
                    ('tfidf', TfidfTransformer()),
                    #('tfidf',TfidfVectorizer()),
                    #("svc", svm.SVC(decision_function_shape='ovo')),
                    ('clf', SGDClassifier(loss='log', penalty='l2',alpha=1e-3, random_state=10, max_iter=10, tol=None)),
                ])
    sgd.fit(X, y)


    #y_pred = sgd.predict(X_test)
    
    #t = sgd.predict_proba(["of electronics department"])
    #print(t) 
    #print(sgd.predict(["what is the eligblity crieteria for addmisson in somaiya "]))   
    #print('accuracy %s' % accuracy_score(y_pred, y_test))
    joblib.dump(sgd, 'intent_model_1.joblib') 
    #print(classification_report(y_test, y_pred,target_names=my_tags))


#train_intent()

'''
calender = 0
faculty =1 
infra = 2
placement = 4
result = 5
small_talk = 6
student body = 7
syllabus = 8

'''

[((18, 9, 18, 62), 'pandas.read_csv', 'pd.read_csv', ({(18, 21, 18, 61): '"""Training Data/intent_training_data.csv"""'}, {}), "('Training Data/intent_training_data.csv')", True, 'import pandas as pd\n'), ((59, 4, 59, 45), 'joblib.dump', 'joblib.dump', ({(59, 16, 59, 19): 'sgd', (59, 21, 59, 44): '"""intent_model_1.joblib"""'}, {}), "(sgd, 'intent_model_1.joblib')", False, 'import joblib\n'), ((34, 15, 34, 49), 're.sub', 're.sub', ({(34, 22, 34, 37): '"""[^a-zA-Z0-9]"""', (34, 39, 34, 42): '""" """', (34, 44, 34, 48): 'data'}, {}), "('[^a-zA-Z0-9]', ' ', data)", False, 'import re\n'), ((44, 29, 44, 46), 'sklearn.feature_extraction.text.CountVectorizer', 'CountVectorizer', ({}, {}), '()', False, 'from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer\n'), ((45, 30, 45, 48), 'sklearn.feature_extraction.text.TfidfTransformer', 'TfidfTransformer', ({}, {}), '()', False, 'from sklearn.feature_extraction.text import TfidfTransformer\n'), ((48, 28, 48, 118), 'sklearn.linear_model.SGDClassifier', 'SGDClassifier', (), '', False, 'from sklearn.linear_model import SGDClassifier\n')]