import numpy as np
import pandas as pd
import catboost as catb
from sklearn.ensemble import RandomForestRegressor
import shutil
import os
"""
# Make some preparations
"""
data_train = pd.read_csv('/kaggle/input/commonlitreadabilityprize/train.csv')
data_test  = pd.read_csv('/kaggle/input/commonlitreadabilityprize/test.csv')
submission = pd.read_csv("/kaggle/input/commonlitreadabilityprize/sample_submission.csv")
y = data_train['target']
error = data_train['standard_error']
data_train
data_test
data_train = data_train.drop('target', axis=1)
data_train = data_train.drop('standard_error', axis=1)
data_train
data_test
all_data = pd.DataFrame(np.vstack((data_train, data_test)))
all_data
import nltk
from nltk.tokenize import sent_tokenize, word_tokenize
#nltk.download('stopwords')
from nltk.corpus import stopwords
from sklearn.feature_extraction.text import TfidfVectorizer
import re
stopword = stopwords.words('english')
wn = nltk.WordNetLemmatizer()
import gensim
from gensim.models import Word2Vec
from sklearn.feature_extraction.text import CountVectorizer
from xgboost import XGBRegressor
import string
string.punctuation
def remove_punct(text):
    text_nopunct = "".join([char for char in text if char not in string.punctuation])
    return text_nopunct

def tokenize_(text):
    tokens = re.split('\W+', text)
    return tokens

def remove_stopwords(tok_list):
    text = [word for word in tok_list if word not in stopword]
    return text

def lemmatizing(tok_text):
    text = [wn.lemmatize(word) for word in tok_text]
    return text

def joini(text):
    words = [" ".join(text) for word in text]
    return words
%%time
all_data[3]  = pd.DataFrame(all_data[3].apply(lambda x: remove_punct(x)))
all_data[3]  = all_data[3].apply(lambda x: tokenize_(x.lower()))
all_data[3]  = all_data[3].apply(lambda x: remove_stopwords(x))
all_data[3]  = all_data[3].apply(lambda x: lemmatizing(x)) 
#all_data[3]  = all_data[3].apply(lambda x: joini(x)) 
for i in range(len(all_data[3])):
    all_data[3][i] = joini(all_data[3][i])
for i in range(len(all_data[3])):
    all_data[3][i] = all_data[3][i][0]
all_data[3][2]
Tfidf_vectorizer = TfidfVectorizer(stop_words=stopwords.words('english'))
all_data_vect = pd.DataFrame(Tfidf_vectorizer.fit_transform(all_data[3]).toarray())
all_data_vect
X = all_data_vect[:2834]
X_test = all_data_vect[2834:2841]
%%time
catboost_est = catb.CatBoostRegressor(task_type="GPU")
#est = RandomForestRegressor()
#catboost_est.fit(X, y)
catboost_est.fit(X, y)
catboost_est.predict(X_test)
#est.predict(X_test)
submission['target'] = catboost_est.predict(X_test)
submission.to_csv("submission.csv", index=False)
submission
#shutil.rmtree("/kaggle/working/catboost_info")
os.remove("kaggle/working/catboost_info/time_left.tsv")
os.remove("kaggle/working/catboost_info/learn_error.tsv")
os.remove("kaggle/working/catboost_info/learn/events.out.tfevents")
os.remove("kaggle/working/catboost_info/catboost_training.json")