# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# +
from sklearn.model_selection import GridSearchCV, RandomizedSearchCV
from scipy.stats import uniform
import sys
import os
import pandas as pd
import numpy as np

import matplotlib.pyplot as plt
import seaborn as sns
from IPython.display import display
# %matplotlib inline

import plotly.offline as py
import plotly.graph_objs as go
import plotly.tools as tls
py.init_notebook_mode(connected=True)

from pandas import set_option
#from pandas.tools.plotting import scatter_matrix
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
from sklearn.model_selection import KFold
from sklearn.model_selection import StratifiedKFold
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import GridSearchCV
from sklearn.pipeline import Pipeline
from xgboost import XGBRegressor
from lightgbm import LGBMRegressor
from sklearn.ensemble import ExtraTreesRegressor, RandomForestRegressor, AdaBoostRegressor, GradientBoostingRegressor

# Metrics
from sklearn.metrics import mean_absolute_error, mean_squared_error, r2_score


import warnings
warnings.filterwarnings('ignore')
# -

# Always good to set a seed for reproducibility
SEED = 7
np.random.seed(SEED)


class RandomSearch(object):
    
    def __init__(self,X_train,y_train,model,hyperparameters):
        
        self.X_train = X_train
        self.y_train = y_train
        self.model = model
        self.hyperparameters = hyperparameters
        
    def RandomSearch(self):
        # Create randomized search 10-fold cross validation and 100 iterations
        cv = 10
        clf = RandomizedSearchCV(self.model,
                                 self.hyperparameters,
                                 random_state=1,
                                 n_iter=150,
                                 cv=cv,
                                 verbose=0,
                                 n_jobs=-1,
                                 )
        # Fit randomized search
        best_model = clf.fit(self.X_train, self.y_train)
        message = (best_model.best_score_, best_model.best_params_)
        print("Best: %f using %s" % (message))

        return best_model,best_model.best_params_
    
    def BestModelPridict(self,X_test):
        
        best_model,_ = self.RandomSearch()
        pred = best_model.predict(X_test)
        return pred
class GridSearch(object):
    
    def __init__(self,X_train,y_train,model,hyperparameters):
        
        self.X_train = X_train
        self.y_train = y_train
        self.model = model
        self.hyperparameters = hyperparameters
        
    def GridSearch(self):
        # Create randomized search 10-fold cross validation and 100 iterations
        cv = 10
        clf = GridSearchCV(self.model,
                                 self.hyperparameters,
                                 cv=cv,
                                 verbose=0,
                                 n_jobs=-1,
                                 )
        # Fit randomized search
        best_model = clf.fit(self.X_train, self.y_train)
        message = (best_model.best_score_, best_model.best_params_)
        print("Best: %f using %s" % (message))

        return best_model,best_model.best_params_
    
    def BestModelPridict(self,X_test):
        
        best_model,_ = self.GridSearch()
        pred = best_model.predict(X_test)
        return pred


df = pd.read_csv("./Data/usedCleaned2021V.csv")

df_enc = df

df_enc

# +
from sklearn.preprocessing import OneHotEncoder

#df_enc = pd.read_csv("./Data/usedCleaned20.csv")
X_t =  df_enc.drop(columns=['Cena', 'Kilometraza', 'Godiste'])
Y_t = df_enc['Cena']

df_enc = pd.get_dummies(df_enc)
X =  df_enc.drop(columns=['Cena', 'Kilometraza', 'Godiste'])
Y = df_enc['Cena']


X_train, X_test, y_train, y_test =train_test_split(X,Y,
                                                   test_size=0.3,
                                                   random_state=0)
X_train_t, X_test_t, y_train_t, y_test_t =train_test_split(X_t,Y_t,
                                                   test_size=0.3,
                                                   random_state=0)
# -

X_train.iloc[0]

X_train_t.iloc[0]

X_train

# +
## Hyperparameters tuning

# +
# model
model = XGBRegressor()

# A parameter grid for XGBoost
hyperparameters = {
        'gamma':[i/10.0 for i in range(0,4)],
        'n_estimators':range(200,1501,200),
        'max_depth':range(7,16,2),
        'min_child_weight':[4,6,8, 10],
        'subsample':[i/100.0 for i in range(75,100,5)],
        'colsample_bytree':[i/100.0 for i in range(65,90,5)],
        'reg_alpha':[0.05]
        }
print('*'*30)
print('XGBRegressor')
print('*'*30)
XGB_RandSearch = RandomSearch(X_train,y_train,model,hyperparameters)
Prediction_XGB = XGB_RandSearch.BestModelPridict(X_test)

#{'subsample': 0.8, 'reg_alpha': 0.05, 'min_child_weight': 6, 'max_depth': 11, 'gamma': 0.2, 'colsample_bytree': 0.75} {'subsample': 0.8, 'reg_alpha': 0.05, 'min_child_weight': 6, 'max_depth': 11, 'gamma': 0.2, 'colsample_bytree': 0.75}
# -

param = {'subsample': 0.95, 'reg_alpha': 0.05, 'n_estimators': 400, 'min_child_weight': 6, 'max_depth': 7, 'gamma': 0.0, 'colsample_bytree': 0.75}
model1 = XGBRegressor(**param)#**param

model1.fit(X_train, y_train)


predictions = model1.predict(X_test)
y_test= inv_boxcox(y_test,lam_price)
predictions = inv_boxcox(predictions,lam_price)

print(r2_score(y_test,predictions))
#print(mean_a(y_test_c,p_sl))

from sklearn.metrics import mean_absolute_error
print(mean_absolute_error(y_test,predictions))

X_test.shape, y_test.shape

df_out = X_test_t
df_out['Cena_actual'] = y_test
df_out['Cena_predicted'] = predictions

df_out['MAE'] = abs(df_out['Cena_actual']-df_out['Cena_predicted'])

df_out.nlargest(n=100, columns='MAE')

df_out['Model'].value_counts()

df[(df['Model']=='C 200') & (df['Godiste']>2014)]

import pickle
with open('ml_model21.pkl', 'wb') as files:
    pickle.dump(model1, files)

# model
model = LGBMRegressor()
hyperparameters = {
    'n_estimators':range(200, 801,200),
    'num_leaves': [60, 80, 120],
    'reg_alpha': [0.1, 0.5],
    'min_data_in_leaf': [30, 50, 70],
    'lambda_l1': [0, 1, 1.5]
    #'lambda_l2': [0, 1]
    }
print('*'*30)
print('LGBMRegressor')
print('*'*30)
LGBM_RandSearch = RandomSearch(X_train,y_train,model,hyperparameters)
Prediction_LGBM = LGBM_RandSearch.BestModelPridict(X_test)
#params = {'reg_alpha': 0.1, 'num_leaves': 60, 'n_estimators': 400, 'min_data_in_leaf': 30, 'lambda_l2': 0, 'lambda_l1': 0}



# +
from sklearn.decomposition import PCA
from sklearn.preprocessing import MinMaxScaler
sc = MinMaxScaler()
X_train = sc.fit_transform(X_train)
X_test = sc.transform(X_test)

pca = PCA()
X_train = pca.fit_transform(X_train)
X_test = pca.transform(X_test)
explained_variance = pca.explained_variance_ratio_
# -

PC_values = np.arange(pca.n_components_) + 1
plt.plot(PC_values, pca.explained_variance_ratio_, 'o-', linewidth=2, color='blue')
plt.title('Scree Plot')
plt.xlabel('Principal Component')
plt.ylabel('Variance Explained')
plt.show()

# +
import matplotlib.pyplot as plt
plt.rcParams["figure.figsize"] = (12,6)

fig, ax = plt.subplots()
xi = np.arange(1, 176, step=1)
y = np.cumsum(pca.explained_variance_ratio_)

plt.ylim(0.0,1.1)
plt.plot(xi, y, marker='o', linestyle='--', color='b')

plt.xlabel('Number of Components')
plt.xticks(np.arange(0, 175, step=10)) #change from 0-based array index to 1-based human-readable label
plt.ylabel('Cumulative variance (%)')
plt.title('The number of components needed to explain variance')

plt.axhline(y=0.95, color='r', linestyle='-')
plt.text(0.5, 0.85, '95% cut-off threshold', color = 'red', fontsize=16)

ax.grid(axis='x')
plt.show()
# -

pca = PCA(n_components=20)
X_train = pca.fit_transform(X_train)
X_test = pca.transform(X_test)

param = {'reg_alpha': 0.1, 'num_leaves': 60, 'n_estimators': 400, 'min_data_in_leaf': 30, 'lambda_l1': 0}
model1 = LGBMRegressor(**param)
model1.fit(X_train, y_train)


predictions = model1.predict(X_test)

print(r2_score(y_test,predictions))
#print(mean_a(y_test_c,p_sl))

from sklearn.metrics import mean_absolute_error
print(mean_absolute_error(y_test,predictions))

import pickle
with open('./models/model_lgb21.pkl', 'wb') as files:
    pickle.dump(model1, files)

# +
# model
model = ExtraTreesRegressor()
hyperparameters = {
    #'criterion': ['mae'],
    'max_depth': [2,8,16],
    'n_estimators': range(50,1500,25),
    'max_features': range(50,401,50),
    'min_samples_leaf': range(20,50,5),
    'min_samples_split': range(15,36,5),
}

print('*'*30)
print('ExtraTreesRegressor')
print('*'*30)
#Extra_RandSearch = RandomSearch(X_train,y_train,model,hyperparameters)
#Prediction_Extra = Extra_RandSearch.BestModelPridict(X_test)
#{'n_estimators': 850, 'min_samples_split': 20, 'min_samples_leaf': 25, 'max_features': 100, 'max_depth': 16}
# -



# +
# Number of trees in random forest
n_estimators = [int(x) for x in np.linspace(start = 200, stop = 2000, num = 200)]
# Number of features to consider at every split
max_features = ['auto', 'sqrt']
# Maximum number of levels in tree
max_depth = [int(x) for x in np.linspace(10, 110, num = 11)]
max_depth.append(None)
# Minimum number of samples required to split a node
min_samples_split = [2, 5, 10]
# Minimum number of samples required at each leaf node
min_samples_leaf = [1, 2, 4]
# Method of selecting samples for training each tree
bootstrap = [True, False]
# Create the random grid
hyperparameters = {'n_estimators': n_estimators,
               'max_features': max_features,
               'max_depth': max_depth,
               'min_samples_split': min_samples_split,
               'min_samples_leaf': min_samples_leaf,
               'bootstrap': bootstrap}
# model
model = RandomForestRegressor()

print('*'*30)
print('RandomForestRegressor')
print('*'*30)
#RandSearch = RandomSearch(X_train,y_train,model,hyperparameters)
#Prediction_Extra = RandSearch.BestModelPridict(X_test)

# +
from sklearn.ensemble import GradientBoostingRegressor


model = GradientBoostingRegressor()

hyperparameters = {
    'n_estimators':range(500, 2001,200),
    'max_depth':range(5,18,2), 
    'min_samples_split':range(100,501,100), 
    'min_samples_leaf':range(30,101,10),
    'max_features':range(16,24,2),
    'subsample':[0.8, 0.85,0.87],
    'learning_rate': [0.001, 0.01, 0.0001]
}


print('*'*30)
print('GradientBoostingRegressor')
print('*'*30)

GB_RandSearch = RandomSearch(X_train,y_train,model,hyperparameters)
Prediction_GB = GB_RandSearch.BestModelPridict(X_test)
#{'subsample': 0.87, 'n_estimators': 1700, 'min_samples_split': 100, 'min_samples_leaf': 40, 'max_features': 22, 'max_depth': 15, 'learning_rate': 0.01}
# -

param = {'subsample': 0.87, 'n_estimators': 1700, 'min_samples_split': 100, 'min_samples_leaf': 40, 'max_features': 22, 'max_depth': 15, 'learning_rate': 0.01}
model1 = GradientBoostingRegressor(**param)
model1.fit(X_train, y_train)
predictions = model1.predict(X_test)
print(r2_score(y_test,predictions))

print(mean_absolute_error(y_test,predictions))


def get_models():
    """Generate a library of base learners."""
    param = {'subsample': 0.95, 'reg_alpha': 0.05, 'n_estimators': 400, 'min_child_weight': 6, 'max_depth': 7, 'gamma': 0.0, 'colsample_bytree': 0.75}
    model1 = XGBRegressor(**param)

    param = {'reg_alpha': 0.1, 'num_leaves': 60, 'n_estimators': 400, 'min_data_in_leaf': 30}
    model2 = LGBMRegressor(**param)
    
    param = {'n_estimators': 850, 'min_samples_split': 20, 'min_samples_leaf': 25, 'max_features': 0.85, 'max_depth': 16}
    model3 = ExtraTreesRegressor(**param)
    

    models = {'XGM':model1, 'LGBM':model2,
              'ETR':model3
              }

    return models

# +

base_learners = get_models()
params = {'subsample': 0.87, 'n_estimators': 1700, 'min_samples_split': 100, 'min_samples_leaf': 40, 'max_features': 22, 'max_depth': 15, 'learning_rate': 0.01, 'random_state':7}
meta_learner = GradientBoostingRegressor(n_estimators=1700,
    max_features=0.7,
    max_depth=13,
    subsample=0.85,
    learning_rate=0.01, 
    random_state=7)

# +
from mlens.ensemble import SuperLearner

# Instantiate the ensemble with 10 folds
sl = SuperLearner(
    folds=10,
    random_state=7,
    verbose=2,
    backend="multiprocessing"
)

# Add the base learners and the meta learner
sl.add(list(base_learners.values())) 
sl.add_meta(meta_learner)

# Train the ensemble
sl.fit(X_train, y_train)

# Predict the test set
p_sl = sl.predict(X_test)
# -

print(r2_score(y_test,p_sl))

from sklearn.metrics import mean_absolute_error
print(mean_absolute_error(y_test,p_sl))

import pickle
with open('ml_modelSuper.pkl', 'wb') as files:
    pickle.dump(sl, files)

#Plot Real vs Predict
plt.figure(figsize=(10,10))
plt.scatter(y_test, predictions, color='blue', edgecolors=(0, 0, 1),alpha=0.5)
plt.plot([y_test.min(), y_test.max()], [y_test.min(), y_test.max()], lw=4,color='Red')
plt.xlabel('Stvarne vrednosti')
plt.xticks(np.arange(1000, 20000, 1000), rotation=60)
plt.yticks(np.arange(1000, 20000, 1000))
plt.ylabel('Predviđene')
plt.show()



StarcoderdataJupyterScriptsDedupFiltered