"""
# Song Popularity Prediction Notebook
"""
#Load the liberaries

import numpy as np
import pandas as pd 
pd.set_option('display.max_columns' ,500)
import matplotlib.pyplot as plt
import seaborn as sns



import os
for dirname, _, filenames in os.walk('/kaggle/input'):
    for filename in filenames:
        print(os.path.join(dirname, filename))

# You can write up to 20GB to the current directory (/kaggle/working/) that gets preserved as output when you create a version using "Save & Run All" 
# You can also write temporary files to /kaggle/temp/, but they won't be saved outside of the current session
#Helper functions code


def investigate_continues_variable(df , column , bins =30 ):
    """
    Function to devide the continues varibales into equal bins 
    then plot a chart showing the 
    counts of popular and un-popular songs 
    accross deffrinet continues numrical variables
    and line chart showing the percentage of the popularity across the selected bins
    
    input : 
        original data frame  -pandas df
        column name - str
        bins number  - int
    
    output :
        chart - matplotlib.pyplot
    
    """
    
    
    #Create new data frame using cross tap between the song popularity and the variables bins
    df_plot = df.copy()
    df_plot =df_plot.reset_index()
    df_plot['bins'] = pd.cut(df_plot[column] , bins = bins ,
                                  labels=range(0,bins))
    df_plot1 = pd.DataFrame(pd.crosstab(df_plot['bins'],df_plot['song_popularity']),)
    df_plot1['Popularity_Percenatge'] = df_plot1[1] / (df_plot1[1] + df_plot1[0]) * 100
    
    
    
    
    #plot the distribution using matplotlib.pyplot
    ind = np.arange(bins)
    width = 0.3

    fig, ax1 = plt.subplots(figsize = (15 ,5))
    ax2 = ax1.twinx()
    ax2.plot(df_plot1.index ,  df_plot1.Popularity_Percenatge  ,label='Percentage of Popularity'  ,
                   ls =':' , lw = 2 , color = 'red' , zorder = 100)
    
    ax2.set_ylim(0,100)
    ax2.set_ylabel("Percentage of Popularity" , color = 'red' , fontsize = 12 , fontfamily = 'serif' , fontweight = 'bold')
    ax2.tick_params(axis='y', colors='red')
    ax2.spines['right'].set_color('red')
    ax2.spines['top'].set_visible(False)
    
    #ax1 setup
    ax1.set_title(f'Percentage of songs Popularity Accross {column}'.title() , fontsize = 12 , fontfamily = 'serif' , fontweight = 'bold')
    ax1.set_xlim(1,bins-1)

    ax1.bar(ind, df_plot1[1] , width, label='Counts of  Popular songs' , color  = '#23a108' ,
                  edgecolor  = 'black' ,hatch =  '*')
    ax1.bar(ind+width, df_plot1[0] , width , label='Counts of un-Popular Songs' , color  = '#8ea389' ,
                  edgecolor  = 'black' )

    ax1.set_xlabel(f"Range of {column} from {round(df[column].min() , 3)} to {round(df[column].max(),3)} divided to {bins} equivilant Bins" ,
                  fontsize = 12 , fontfamily = 'serif' , fontweight = 'bold')
    ax1.set_ylabel("Counts Per each Bin" , fontsize = 12 , fontfamily = 'serif' , fontweight = 'bold')
    ax1.spines['top'].set_visible(False)
    ax1.spines['right'].set_color('red')
    ax1.set_xticks([])
    
    fig.legend( bbox_to_anchor=(0.2, 1.02, 0.6, .102), loc='lower left',
                      ncol=3, mode="expand", borderaxespad=0.)
    plt.show()
    


    
def investigate_discrete_Variables (df , column):
    """
    Function to plot a bar chart showing the 
    counts of popular and un-popular songs 
    accross deffrinet continues numrical variables
    
    input : 
        original data frame  -pandas df
        column name - str
    
    output :
        chart - matplotlib.pyplot
    
    
    """
    new = df.groupby([column , 'song_popularity']).size().reset_index()
    pop = new[new.song_popularity == 1]
    unpop = new[new.song_popularity  == 0]
    
    pop[column].apply(str)
    unpop[column].apply(str)
    
    
    ind = np.arange(len(pop[column]))
    width = 0.3


    fig, ax1 = plt.subplots(figsize = (15 ,5))
    
    #ax1 setup
    ax1.set_title(f'Counts of Popular and non-Popular songs Accross {column}'.title() , fontsize = 12 , fontfamily = 'serif' , fontweight = 'bold')

    ax1.bar(ind, pop[0] , width, label='Counts of  Popular songs' , color  = '#23a108' ,
                  edgecolor  = 'black' ,hatch =  '*')
   
    ax1.bar(ind+width, unpop[0] , width , label='Counts of un-Popular Songs' , color  = '#8ea389' ,
                  edgecolor  = 'black' )

    

    ax1.set_ylabel("Counts " , fontsize = 12 , fontfamily = 'serif' , fontweight = 'bold')
    ax1.spines['top'].set_visible(False)
    ax1.spines['right'].set_visible(False)
    
    ax1.set_xticks(pop[column])
    
    fig.legend( bbox_to_anchor=(0.2, 1.02, 0.6, .102), loc='lower left',
                      ncol=3, mode="expand", borderaxespad=0.)
    ax1.grid(axis = 'y' , zorder =-1 , linestyle =":")
    plt.show()

train = pd.read_csv('/kaggle/input/song-popularity-prediction/train.csv')
test = pd.read_csv('/kaggle/input/song-popularity-prediction/test.csv')
test['song_popularity'] = np.nan
df = pd.concat([train, test])

display(df)
"""

!pip install sweetviz
import sweetviz
report = sweetviz.analyze([train,'df'],target_feat='song_popularity')
report.show_html('report.html')

"""
print('Pass')
"""
# Data Cleaning Pipe Line 
"""

#Reload the Data frame
df = pd.concat([train, test])

#df.id
df.drop(['id' , 'instrumentalness' ] ,axis  = 1 , inplace = True)


#df.song_duration_ms
df.song_duration_ms.fillna(df.song_duration_ms.mean() , inplace = True)
df.song_duration_ms = df.song_duration_ms.apply(lambda x: df.tempo.mean() if x >400000 else x)

#df.acousticness
from sklearn.preprocessing import PowerTransformer
Power_Transfrom_acous = PowerTransformer(method='yeo-johnson')
df.acousticness = Power_Transfrom_acous.fit_transform(df.acousticness.values.reshape(-1, 1))
df.acousticness.fillna(df.acousticness.mode()[0] , inplace = True)


#df.danceability
df.danceability.fillna(df.danceability.mean() , inplace = True)


#df.energy
df.energy.fillna(df.energy.mean() , inplace = True)


#df.liveness
df.liveness = df.liveness.apply(lambda x: df.liveness.mean() if x>0.75 else x)
Power_Transfrom_liv = PowerTransformer(method='yeo-johnson')
df.liveness = Power_Transfrom_liv.fit_transform(df.liveness.values.reshape(-1, 1))
df.liveness.fillna(df.liveness.mode()[0] , inplace = True)

#df.loudness
Power_Transfrom_loud = PowerTransformer(method='yeo-johnson')
df.loudness= Power_Transfrom_loud.fit_transform(df.loudness.values.reshape(-1, 1))
df.loudness.fillna(df.loudness.mean() , inplace = True)



#df.speechiness
df.speechiness = df.speechiness.apply(lambda x: df.speechiness.mean() if x>0.45 else x)
Power_Transfrom_speac= PowerTransformer(method='yeo-johnson')
df.speechiness = Power_Transfrom_speac.fit_transform(df.speechiness.values.reshape(-1, 1))
df.speechiness.fillna(df.speechiness.mode()[0] , inplace = True)


#df.tempo
df.tempo = df.tempo.apply(lambda x: df.tempo.mean() if x >190 else x)
df.tempo.fillna(df.tempo.mean() , inplace = True)

#df.audio_valence
df.audio_valence.fillna(df.audio_valence.mean() , inplace = True)

#df.key
df.key.fillna(12 , inplace = True)



"""
# Checking the outliers after data cleaning
"""
fig , ax = plt.subplots(3,4 , figsize = (10,10))
for column  , axes in zip(df.columns , ax.flatten()):
    axes.boxplot(df[column] , patch_artist=True , boxprops=dict(facecolor='#23a108', edgecolor  = 'black' ,hatch =  '*'))
    axes.set_title(column)

plt.tight_layout()
plt.show()
"""


!pip install sweetviz
import sweetviz
train = df[df.song_popularity.notnull()]
report = sweetviz.analyze([train,'df'],target_feat='song_popularity')
report.show_html('report.html')

"""
print ('pass')
"""
# Visualize the continues numirical variables
"""
Continues_numerical = ['song_duration_ms','acousticness','danceability', 'energy',
                       'liveness', 'loudness', 'speechiness', 'tempo', 'audio_valence']



for column in Continues_numerical:
    investigate_continues_variable(df , column  , bins = 25)

popular = []
unpopular = []
for column in Continues_numerical:
    popular.append(df[df.song_popularity == 1][column].mean())
    unpopular.append(df[df.song_popularity == 0][column].mean())

ind = 1
width = 0.3  
fig , ax = plt.subplots(3,3 , figsize  = (15 ,15))
for column , pop , unpop , axes in zip (Continues_numerical , popular , unpopular , ax.flatten()):
    
    axes.bar(ind, pop , width, label='Average of  Popular songs' , color  = '#23a108' ,
                  edgecolor  = 'black' ,hatch =  '*' )
    axes.bar(ind+width+0.1, unpop , width , label='Average un-Popular Songs' , color  = '#8ea389' ,
                  edgecolor  = 'black' )
    axes.set_title(column ,fontsize = 12 , fontfamily = 'serif' , fontweight = 'bold' )
    axes.spines['top'].set_visible(False)
    axes.spines['right'].set_visible(False)
    axes.set_xticks([])

    
font = {'family':'serif','weight':'bold','style':'normal', 'size':12}

fig.suptitle('Compasion between the average of Continues Variables accross the popularity'.title(), 
             fontsize = 16 , fontfamily = 'serif' , fontweight = 'bold')
ax[0,0].legend(bbox_to_anchor=(0.2, 1.2, 3, .102), loc='lower left',
                      ncol=2, mode="expand", borderaxespad=0. , prop = font)
plt.show();
"""
# Inverstigate the Descrete columns
"""
descrite_numerical = [ 'key','audio_mode','time_signature']
for column in descrite_numerical:
    investigate_discrete_Variables (df , column)
"""
# Check the coorrelation with the song popularity
"""
corr_df = df.drop('song_popularity', axis=1).corrwith(df.song_popularity).sort_values().reset_index().rename(columns = {'index':'feature' ,0:'correlation'})

fig , ax = plt.subplots(figsize  = (5,8))
ax.barh(y =corr_df.feature , width = corr_df.correlation ,
        color  = '#23a108' ,edgecolor  = 'black' ,hatch =  '*' )
ax.set_title('correlation between featuer and target'.title() ,
            fontsize = 16 , fontfamily = 'serif' , fontweight = 'bold')
plt.show();
df.drop(['tempo' , 'audio_mode' ,'song_duration_ms',  'key' ] , axis = 1 , inplace = True)
"""
# Try PCA and clustring of the train data
"""
np.random.seed(42)


train = df[df.song_popularity.notnull()]

from sklearn.preprocessing import StandardScaler
SC = StandardScaler()

train_scale = SC.fit_transform(train)


from sklearn.decomposition import PCA
pca = PCA(n_components=3)
princebal_componants_array = pca.fit_transform(train_scale)

from sklearn.cluster import KMeans
model42 = KMeans(n_clusters=10 , random_state=42)
model42.fit(princebal_componants_array)

labels = model42.labels_

train ['labels'] = labels

    
    
corr_df = train.drop('song_popularity', axis=1).corrwith(train.song_popularity).sort_values().reset_index().rename(columns = {'index':'feature' ,0:'correlation'})
fig , ax = plt.subplots(figsize  = (5,8))
ax.barh(y =corr_df.feature , width = corr_df.correlation ,
        color  = '#23a108' ,edgecolor  = 'black' ,hatch =  '*' )
ax.set_title('correlation between featuer and target after adding the labels'.title() ,
            fontsize = 16 , fontfamily = 'serif' , fontweight = 'bold')
plt.show();
def investigate_continues_variable_7_2 (df ,label ,  column , bins =30 , ax1=None ):
    """
    Function to devide the continues varibales into equal bins 
    then plot a chart showing the 
    counts of popular and un-popular songs 
    accross deffrinet continues numrical variables
    and line chart showing the percentage of the popularity across the selected bins
    
    input : 
        original data frame  -pandas df
        column name - str
        bins number  - int
    
    output :
        chart - matplotlib.pyplot
    
    """
    
    
    #Create new data frame using cross tap between the song popularity and the variables bins
    df_plot = df.copy()
    df_plot =df_plot.reset_index()
    df_plot = df_plot[df_plot.labels == label]
    df_plot['bins'] = pd.cut(df_plot[column] , bins = bins ,
                                  labels=range(0,bins))
    df_plot1 = pd.DataFrame(pd.crosstab(df_plot['bins'],df_plot['song_popularity']),)
    df_plot1['Popularity_Percenatge'] = df_plot1[1] / (df_plot1[1] + df_plot1[0]) * 100
    
    
    
    
    #plot the distribution using matplotlib.pyplot
    ind = np.arange(bins)
    width = 0.3

    
    ax2 = ax1.twinx()
    ax2.plot(df_plot1.index ,  df_plot1.Popularity_Percenatge  ,label='Percentage of Popularity'  ,
                   ls =':' , lw = 2 , color = 'red' , zorder = 100)
    
    ax2.set_ylim(0,100)
    ax2.set_ylabel("Percentage of Popularity" , color = 'red' , fontsize = 12 , fontfamily = 'serif' , fontweight = 'bold')
    ax2.tick_params(axis='y', colors='red')
    ax2.spines['right'].set_color('red')
    ax2.spines['top'].set_visible(False)
    
    #ax1 setup
    ax1.set_title(f'Percentage of songs Popularity Accross {column} Label number {label}'.title() , fontsize = 12 , fontfamily = 'serif' , fontweight = 'bold')
    ax1.set_xlim(1,bins-1)

    ax1.bar(ind, df_plot1[1] , width, label='Counts of  Popular songs' , color  = '#23a108' ,
                  edgecolor  = 'black' ,hatch =  '*')
    ax1.bar(ind+width, df_plot1[0] , width , label='Counts of un-Popular Songs' , color  = '#8ea389' ,
                  edgecolor  = 'black' )

    ax1.set_xlabel(f"Range of {column} from {round(df[column].min() , 3)} to {round(df[column].max(),3)} divided to {bins} equivilant Bins" ,
                  fontsize = 12 , fontfamily = 'serif' , fontweight = 'bold')
    ax1.set_ylabel("Counts Per each Bin" , fontsize = 12 , fontfamily = 'serif' , fontweight = 'bold')
    ax1.spines['top'].set_visible(False)
    ax1.spines['right'].set_color('red')
    ax1.set_xticks([])
    
    
    
train ['labels'] = labels
for label in train.labels.unique().tolist():
    print (label, " :  mean of popularity " , round(train[train.labels ==label].song_popularity.mean() , 2) , " & Count per label " , train[train.labels ==label].song_popularity.count())
    print ('_________________________')
train.labels = train.labels.apply(lambda x: 100 if x in [2, 9 , 5 , 4 ] else x)
train.labels = train.labels.apply(lambda x: 200 if x in [0 , 3 , 8 ] else x)
train.labels = train.labels.apply(lambda x: 300 if x in [1 , 7 , 6 ] else x)
for label in train.labels.unique().tolist():
    print ("label number" ,label, " :  mean of song popularity " , round(train[train.labels ==label].song_popularity.mean() , 2) , " & Count per label " , train[train.labels ==label].song_popularity.count())
    print ('_________________________')
fig , ax = plt.subplots(2,2 , figsize = (15,10))



investigate_continues_variable_7_2 (train ,100 ,  column = 'energy' , bins =15 , ax1=ax[0,0] )
investigate_continues_variable_7_2 (train ,300 ,  column = 'energy' , bins =15 , ax1=ax[0,1] )
investigate_continues_variable_7_2 (train ,100,  column = 'speechiness' , bins =15 , ax1=ax[1,0] )
investigate_continues_variable_7_2 (train ,300 ,  column = 'speechiness' , bins =15 , ax1=ax[1,1] )
plt.tight_layout()
"""
# Predict the new labels in the test dataset
"""
train.head()
from sklearn.preprocessing import StandardScaler
SC = StandardScaler()

train['energy_double'] = train.energy * 10
train['speechiness_double'] = train.speechiness *10

train_after_clsuster = train.drop(['song_popularity' , 'labels'] , axis = 1).values
train_after_clsuster = SC.fit_transform(train_after_clsuster)



target = train.labels.values



from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(train_after_clsuster, target, test_size=0.25, random_state=42)



from sklearn.neural_network import MLPClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.gaussian_process.kernels import RBF
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis
from sklearn.metrics import roc_auc_score
from sklearn.metrics import accuracy_score

neural_network = MLPClassifier()
neighbors = KNeighborsClassifier()
svm = SVC()
kernels = RBF()
tree = DecisionTreeClassifier()
RandomForestClassifier = RandomForestClassifier()
AdaBoostClassifier = AdaBoostClassifier()
naive_bayes = GaussianNB()
discriminant_analysis = QuadraticDiscriminantAnalysis()

clfs = [neural_network ,neighbors ,  svm   ]
names = ['neural_network' ,'neighbors' ,  'svm'  ]


for clf , name in zip (clfs , names):
    clf.fit(X_train, y_train)
    predict_train = clf.predict(X_train)
    predict_test = clf.predict(X_test)
    
    print (name)
    print ('train ROC : ' , accuracy_score(y_train , predict_train))
    print ('test ROC : ' , accuracy_score(y_test , predict_test))
    print ('_________________________')
clfs = [tree ,  
        RandomForestClassifier , AdaBoostClassifier , naive_bayes ,discriminant_analysis  ]
names = ['tree' ,  
        'RandomForestClassifier' , 'AdaBoostClassifier' , 'naive_bayes' ,'discriminant_analysis'  ]


for clf , name in zip (clfs , names):
    clf.fit(X_train, y_train)
    predict_train = clf.predict(X_train)
    predict_test = clf.predict(X_test)
    
    print (name)
    print ('train ROC : ' , accuracy_score(y_train , predict_train))
    print ('test ROC : ' , accuracy_score(y_test , predict_test))
    print ('_________________________')
"""
# Adding labels to the test dataset
"""
test = df[df.song_popularity.isnull()].drop('song_popularity' , axis = 1)
test['energy_double'] = test.energy * 10
test['speechiness_double'] = test.speechiness *10
test_scaled = SC.transform(test.values)
test_labels = neural_network.predict(test_scaled)

test['labels'] = test_labels
test.head()
train.head()
"""
# Predict the song popularity according to the new features
"""
"""
# Regressor
"""
X = train.drop('song_popularity' , axis = 1).values
y = train.song_popularity.values
X = SC.fit_transform(X)

from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.25, random_state=42)



import os
import pandas as pd
from pandas import DataFrame,Series
from sklearn import tree
import matplotlib
import numpy as np
import matplotlib.pyplot as plt
from sklearn import svm
from sklearn.preprocessing import StandardScaler
import statsmodels.formula.api as smf
import statsmodels.api as sm
from mpl_toolkits.mplot3d import Axes3D
import seaborn as sns
from sklearn import neighbors
from sklearn import linear_model
from sklearn.neighbors import KNeighborsRegressor
from sklearn.ensemble import RandomForestRegressor
%matplotlib inline


model=linear_model.Ridge()
knn=neighbors.KNeighborsRegressor()
reg = linear_model.BayesianRidge()
dec = tree.DecisionTreeRegressor(max_depth=1)
svm_reg=svm.SVR()
reg_random_forest = RandomForestRegressor()



regs = [model ,knn , reg ,dec  ]
names = ['model' ,'knn' , 'reg' ,'dec'  ]

for reg , name in zip (regs , names):
    reg.fit(X_train, y_train)
    predict_train = reg.predict(X_train)
    predict_test = reg.predict(X_test)
    
    print (name)
    print ('train ROC : ' , roc_auc_score(y_train , predict_train))
    print ('test ROC : ' , roc_auc_score(y_test , predict_test))
    print ('_________________________')
"""
# Classifier Comparasion
"""
from sklearn.neural_network import MLPClassifier
from sklearn.neighbors import KNeighborsClassifier
from sklearn.svm import SVC
from sklearn.gaussian_process.kernels import RBF
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, AdaBoostClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.discriminant_analysis import QuadraticDiscriminantAnalysis
from sklearn.metrics import roc_auc_score

neural_network = MLPClassifier()
neighbors = KNeighborsClassifier()
svm = SVC()
kernels = RBF()
tree = DecisionTreeClassifier()
RandomForestClassifier = RandomForestClassifier()
AdaBoostClassifier = AdaBoostClassifier()
naive_bayes = GaussianNB()
discriminant_analysis = QuadraticDiscriminantAnalysis()

clfs = [neural_network ,neighbors ,  svm   ]
names = ['neural_network' ,'neighbors' ,  'svm'  ]


for clf , name in zip (clfs , names):
    clf.fit(X_train, y_train)
    predict_train = clf.predict(X_train)
    predict_test = clf.predict(X_test)
    
    print (name)
    print ('train ROC : ' , roc_auc_score(y_train , predict_train))
    print ('test ROC : ' , roc_auc_score(y_test , predict_test))
    print ('_________________________')
clfs = [tree ,  
        RandomForestClassifier , AdaBoostClassifier , naive_bayes ,discriminant_analysis  ]
names = ['tree' ,  
        'RandomForestClassifier' , 'AdaBoostClassifier' , 'naive_bayes' ,'discriminant_analysis'  ]


for clf , name in zip (clfs , names):
    clf.fit(X_train, y_train)
    predict_train = clf.predict(X_train)
    predict_test = clf.predict(X_test)
    
    print (name)
    print ('train ROC : ' , roc_auc_score(y_train , predict_train))
    print ('test ROC : ' , roc_auc_score(y_test , predict_test))
    print ('_________________________')
"""
# Submission
"""
test = SC.transform(test)
submision_prediction = knn.predict(test)
submission = pd.read_csv('/kaggle/input/song-popularity-prediction/sample_submission.csv')
submission['song_popularity'] =submision_prediction
submission.to_csv('knn.scv' , index = False)