# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
# %matplotlib inline

# Read in the file here

# +
spotify = pd.read_csv('Data/SpotifyFeatures0419.csv')

spotify.head(2)

# +
#spotify.info()
# -

# #### Restructuring the data

# +
#order the data by the target column; drop the track_id and superfluous index column
spotify_ordered = spotify.sort_values('popularity', ascending=False).reset_index()
so = spotify_ordered.drop(['index', 'track_id'], axis=1)

#start index at 1 to accurately see where popularities rank
so.index = so.index + 1

#drop songs with popularity of zero as they create noise
so = so[so.popularity > 0]
so.tail()
# -

#convert certain columns to categorical as they shouldn't 
#be treated numerically or ordinally
so[['mode', 'key', 'time_signature']] = so[['mode', 'key', 'time_signature']].astype('category')

so.head()

# +
#print(so.dtypes)

# +
#Create new table with a column classifying popularity ratings
classified = so.copy()
classified['pop_rating'] = ''

for i, row in classified.iterrows():
    score = 'unpopular'
    if (row.popularity > 50) & (row.popularity < 75):
        score = 'medium'
    elif row.popularity >= 75:
        score = 'popular'
    classified.at[i, 'pop_rating'] = score
    
classified.head()

# +
#Set up the data for encoding: 'track_name' is categorical but is not useful so it will be dropped
encoded = so.drop('track_name', axis=1)

encoded = pd.get_dummies(encoded)
# -

encoded.shape

# Now there are three separate dataframes to experiment with:<br>
# so: The basic data ordered by popularity<br>
# classified: The data with an added classifier target variable<br>
# encoded: Dataframe where the catagorical variables have been transformed

# +
#experiment with a basic decision tree on the classified data

from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score

#drop all non-numeric columns, also the target variable
X_tree = classified.drop\
(['artist_name', 'track_name', 'key', 'mode', 'time_signature', 'popularity', 'pop_rating'], axis=1)
y_tree = classified.pop_rating

dt = DecisionTreeClassifier(max_depth=20, random_state=7)

X_tr, X_tst, y_tr, y_tst = train_test_split(X_tree, y_tree, test_size=0.3, random_state=5)

dt.fit(X_tr, y_tr)

y_pred = dt.predict(X_tst)
y_pred[:5]
# -

accuracy_score(y_pred, y_tst)

# +
#Try a regressor tree

from sklearn.tree import DecisionTreeRegressor

X_rg = so.drop\
(['artist_name', 'track_name', 'key', 'mode', 'time_signature', 'popularity'], axis=1)
y_rg = so.popularity

X_train, X_test, y_train, y_test = train_test_split(X_rg, y_rg, test_size=0.3, random_state=12)

dt_reg = DecisionTreeRegressor() #try unrestrained

dt_reg.fit(X_train, y_train)

rg_pred = dt_reg.predict(X_test)
rg_pred[:3]
# -

dt_reg.score(X_test, y_test)

# ##### Bagging

# +
from sklearn.ensemble import BaggingClassifier

#splt from previous classifier
dt = DecisionTreeClassifier(max_depth=20, random_state=18)
bc = BaggingClassifier(base_estimator=dt, n_estimators=100, random_state=1)

bc.fit(X_tr, y_tr)

bc_pred = bc.predict(X_tst)

accuracy_score(bc_pred, y_tst)
# -

# ##### Random Forest

# +
from sklearn.ensemble import RandomForestClassifier

rf = RandomForestClassifier(n_estimators=100, random_state=1)

rf.fit(X_tr, y_tr)

rf_pred = rf.predict(X_tst)

accuracy_score(rf_pred, y_tst)

# +
important = pd.Series(data=rf.feature_importances_, index=X_tr.columns).sort_values()

_ = important.plot(kind='barh')
_ = plt.title('RF feature importances')
# -

# It seems surprising at first that duration is such an impactful feature but, after thinking about it, it kind of makes sense. Popular songs do tend to consistently measure around the 3 to 4 minute mark. A 10 minute song isn't exactly 'radio-friendly', and shorter songs are likely easier for the casual listener to digest.<br>
# <br>
# But, on the whole, there isn't a huge differentiation between features.











# ###### sklearn knn test
# <br>
# This was just something I tried in order to practice new material. I now know that this is a classification model, and not suited for what I was trying to accomplish.

'''from sklearn.neighbors import KNeighborsClassifier

X = spotify[['acousticness', 'danceability', 'energy', 'speechiness']].head()
y = spotify.popularity.head()

X_new = spotify[['acousticness', 'danceability', 'energy', 'speechiness']].iloc[6:11, :]

knn = KNeighborsClassifier(n_neighbors=5)
knn.fit(X,y)
'''

# +
#knn.predict(X_new)
# -

# <b>Investigating correlations in the spotify data</b>

# +
#spotify.corr()

# +
import seaborn as sns

#sns.heatmap(spotify.corr(), square=True, cmap='RdYlGn')
# -

# Here's a rudimentary attempt at a linear regression model. This is just for practice and includes a few arbitrary columns.

# +
from sklearn.linear_model import LinearRegression, Ridge
from sklearn.model_selection import cross_val_score, train_test_split

#pick a few independent variables, and set the dependent variable
X = spotify[['acousticness', 'danceability', 'energy', 'liveness', 'loudness', 'speechiness']]
y = spotify.popularity

#fit and predict the model
lm = LinearRegression()
lm.fit(X,y)
lm.predict(X)[:2]
# -

lm.score(X,y)

# The score for this model is pretty poor. Let's adjust the data and try again.

# +
#re-run model for songs with pop greater than 0
over_zero = spotify[spotify['popularity'] > 0]
X = over_zero[['acousticness', 'danceability', 'energy', 'liveness', 'loudness', 'speechiness']]
y = over_zero.popularity

#fit and predict the model
lm = LinearRegression()
lm.fit(X,y)
prediction = lm.predict(X)
prediction[:2]
# -

actual_pop = zip(y, prediction)
actual_100 = list(actual_pop)[:100]
actual_100[:3]

# +
diffs = {'Under 5': 0, '5 and 10': 0, 'Over ten': 0}
for pair in actual_100:
    if abs(pair[0] - pair[1]) < 5:
        diffs['Under 5'] += 1
    elif 5 <= abs(pair[0] - pair[1]) < 10:
        diffs['5 and 10'] += 1
    else:
        diffs['Over ten'] += 1
            
diffs
# -

# The score is slightly improved

lm.score(X,y)

# ### <b>pd.get_dummies</b>
# <br>
# Restructuring the model to account for categorical data.

# +
#Set up the data for encoding: 'track_name' is categorical but is not useful so it will be dropped
encoded = so.drop('track_name', axis=1)

#Now, drop songs with popularity of zero as they create noise
encoded = encoded[encoded['popularity'] > 0]

encoded = pd.get_dummies(encoded)
# -

encoded.shape

# +
# %%time
encoded_samp = encoded.sample(n=20000, replace=True)

X = encoded_samp.drop('popularity', axis=1)
y = encoded_samp.popularity

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)

lm = LinearRegression()
lm.fit(X_train,y_train)
# -

lm.score(X_test, y_test)

# Predict over 10,000 rows and create custom measurement to assess model:

# +
encoded_samp = encoded.sample(n=10000, replace=True)

X = encoded_samp.drop('popularity', axis=1)
y = encoded_samp.popularity

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)

lin = LinearRegression()
lin.fit(X_train,y_train)
# -

lin.score(X_test, y_test)

prd = lin.predict(X_test)


# +
def custom_loss(prediction, actual):
    paired = zip(prediction, actual)
    listed = list(paired)
    diffs = {'Under 5': 0, '5 and 10': 0, 'Over ten': 0}

    for pair in listed:
        if abs(pair[0] - pair[1]) < 5:
            diffs['Under 5'] += 1
        elif 5 <= abs(pair[0] - pair[1]) < 10:
            diffs['5 and 10'] += 1
        else:
            diffs['Over ten'] += 1
     
    return diffs

print(custom_loss(prd, y_test))
# -

print(len(prd), len(y_test))

# +
#percentages of encoded "successes"
x = (743 + 348) / 3000

x

# +
'''Retry model without the encoded categoricals'''
X = over_zero.drop(['artist_name', 'track_id', 'track_name', 'key', 'mode', 'time_signature', 'popularity'], axis=1)
y = over_zero.popularity

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)

lreg = LinearRegression()
lreg.fit(X_train,y_train)
pred = lreg.predict(X_test)

print(custom_loss(pred, y_test))

# +
#X_test.columns
# -

print(lreg.score(X_test, y_test))

# This model has a perfect score? That doesn't seem right.
# <br>
# Okay, nevermind. Once I removed the popularity info from the X variable, the results are more as expected. However, the results are improved over the large dataset with the encoded categorical data. Perhaps that's because the error is less amplified.

#Check which coefficients are best
coeffs = pd.concat([pd.DataFrame(X.columns), pd.DataFrame(lin.coef_)], axis=1)
coeffs.columns = ['Feature', 'Value']
coeffs = coeffs.sort_values('Value', ascending=False)
coeffs.reset_index(inplace=True)
coeffs.drop('index', axis=1).head()


# Fit linear model over batches:

# +
#batching
def batch_score(dt, size):
    scores = []
    for i in range(10):
        sample = dt.sample(n=size, replace=True)
    
        X = sample.drop('popularity', axis=1)
        y = sample.popularity
    
        X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.3, random_state=42)
    
        reg = LinearRegression()
        reg.fit(X_train, y_train)
    
        scores.append(reg.score(X_test, y_test))
    return scores

print(np.median(batch_score(encoded, 10000)))
# -

# <b>A basic ridge model without hyperparameter tuning:</b>

# +
from sklearn.linear_model import Ridge
from sklearn.model_selection import train_test_split

X_r = encoded_samp.drop(['key', 'mode', 'time_signature', 'popularity'], axis=1)
y_r = encoded_samp.popularity

X_tr, X_ts, y_tr, y_ts = train_test_split(X_r, y_r, test_size=0.3, random_state=5)

ridge = Ridge(alpha = 0.5, normalize=True)
ridge.fit(X_tr, y_tr)

ridge.score(X_ts, y_ts)

# +
#Custom loss function results
pred_rdg = ridge.predict(X_ts)

print(custom_loss(pred_rdg, y_ts))
# -

# The ridge model above is definitely an imporvement over the basic linear model using the encoded data.

# +
from sklearn.model_selection import cross_val_score

cv_res = cross_val_score(ridge, X_r, y_r, cv=5)
cv_res

# +
from sklearn.linear_model import Lasso

names = encoded_samp.drop(['key', 'mode', 'time_signature', 'popularity'], axis=1).columns

lasso = Lasso(alpha=0.1)

lasso_coefs = lasso.fit(X_r, y_r).coef_

coef_vals = list(zip(names, (list(lasso_coefs))))

print([pair for pair in coef_vals if abs(pair[1]) > 0])

# +
Xl_train, Xl_test, yl_train, yl_test = train_test_split(X_r, y_r, test_size=0.3, random_state=21)

lasso.fit(Xl_train, yl_train)
lasso.score(Xl_test, yl_test)
# -

lasso_pred = lasso.predict(Xl_test)
print(custom_loss(lasso_pred, yl_test))

# The ridge regression model seems to perform better than the others.

# ## Popular songs to total songs ratio

# Using a histogram and playing around with the binning parameter, I came across an interesting feature of the data: the frequency of song popularities seem to spike in intervals of 25. This is a curious characteristic worth exploring later, but for now I want to focus on the last spike around the 75 mark. I can use this group to explore what is popular, in this case anything with a score of at least 75, and find all artists with at least one song that fits that criteria.

# +
over_0 = so[so['popularity'] > 0]

_ = over_0['popularity'].plot(kind='hist', bins=95)
# -

#Subset the data for only songs with popularity over 75 and count the number of tracks each artist had
#fitting this criteria. This will give a total count of 'popular' songs for each artist.
pop_artists = so[so['popularity'] > 75].groupby('artist_name').count().sort_values('track_name', ascending=False)
pop_artists = pop_artists.iloc[:,:1].sort_index()
pop_artists.columns = ['total_pop_tracks']
pop_artists.head(3)

# Now, create a new dataframe with all the songs, not just the popular songs, from all the artists that have at least one popular song. This will allow us to compare the number of popular songs to the total number of songs.

# +
x = list(pop_artists.index)

pop_full_catalog = pd.DataFrame()
for name in x:
    pop_full_catalog = pop_full_catalog.append(so[so['artist_name'] == name])
    
pop_full_catalog.head(3)
# -

# Count the total number of tracks for each popular artist.

pop_total_count = pop_full_catalog.groupby('artist_name').count().sort_values('track_name', ascending=False)
pop_total_count = pop_total_count.iloc[:,:1].sort_index()
pop_total_count.columns = ['total_tracks']
pop_total_count.head(3)

# Compute the ratio for popular tracks compared to total tracks. Create a new dataframe to show this info.

compare = pd.concat([pop_artists, pop_total_count], axis=1)
compare['popular_ratio'] = compare.total_pop_tracks / compare.total_tracks
compare = compare.sort_values('total_pop_tracks', ascending=False)
compare.head(10)

compare.describe()

compare[compare['total_pop_tracks'] == 1].sort_values('total_tracks', ascending=False)

# <b>Observations:</b>
# <br>
# -Looking at the previous few cells, there seems to be a good bit of variability among total tracks for popular artists, ranging between 1 and 75. The ratio of popular tracks to total tracks also varies widely. <br>
# -There are quite a few artists with a ratio of 100%. <br>
# -The artist with the most popular tracks tend to have fairly high ratios, perhaps reinforcing the idea that the best indicator of creating a new hit song is to already have an old hit song.

# <b>Plot the data to look for correlations.</b>

_ = plt.hist(compare.total_pop_tracks, bins=20)

# +
x = compare.total_pop_tracks
y = compare.popular_ratio

plt.plot(x, y, 'go--')
plt.title('Total Popular Tracks Compared to Ratio')
plt.xlabel('Total popular tracks')
plt.ylabel('Popular ratio')

# +
re_compare = compare.sort_values('total_tracks', ascending=False)
xx = re_compare.total_tracks
yy = re_compare.popular_ratio

plt.plot(xx, yy, 'bo--')
plt.title('Total Tracks Compared to Ratio')
plt.xlabel('Total tracks')
plt.ylabel('Popular ratio')
# -

# Comparing the charts above, having more total tracks tends to lead to a lower popular ratio, while those artists with more total popular songs tend to have a higher popular ratio despite having to put out more total tracks in order to score more 'hits'.

sns.set()
_ = sns.lmplot(x='total_tracks', y='total_pop_tracks', data=compare, scatter_kws={"s": 10})
_ = plt.xlabel('Total tracks', fontsize=14)
_ = plt.ylabel('Total popular tracks', fontsize=14)

# ### K-Means

# +
from sklearn.cluster import KMeans

X = np.array([[5,3],  
     [10,15],
     [15,12],
     [24,10],
     [30,45],
     [85,70],
     [71,80],
     [60,78],
     [55,52],
     [80,91],])

_ = plt.scatter(X[:,0],X[:,1], label='True Position')  
# -

kmeans = KMeans(n_clusters=2)
kmeans.fit(X)
print(kmeans.cluster_centers_)

kmeans.labels_

plt.scatter(X[:,0], X[:,1], c=kmeans.labels_, cmap='Accent')

# Now try it with the popularity target variable with songs that have popularity over zero. Start with three clusters.

# +
X = np.array(over_zero.popularity).reshape(-1,1)

kmeans = KMeans(n_clusters=3)
kmeans.fit(X)

kmeans.cluster_centers_
# -

plt.scatter(X, over_zero.loudness, c=kmeans.labels_, s=12)

plt.scatter(X, over_zero.instrumentalness, c=kmeans.labels_, s=12)

# Increasing the interations doesn't change anything.

# +
X = np.array(over_zero.popularity).reshape(-1,1)

kmeans = KMeans(n_clusters=3, max_iter=600)
kmeans.fit(X)

kmeans.cluster_centers_
# -

# Now try with more clusters to raise the bar for what qualifies as popular.

# +
X = np.array(over_zero.popularity).reshape(-1,1)

kmeans = KMeans(n_clusters=5)
kmeans.fit(X)

kmeans.cluster_centers_
# -







# #### Explore the target column: popularity

spotify['popularity'].describe()

# +
#order the data by the target column; drop the track_id and superfluous index column
spotify_ordered = spotify.sort_values('popularity', ascending=False).reset_index()
so = spotify_ordered.drop(['index', 'track_id'], axis=1)

#start index at 1 to accurately see where popularities rank
so.index = so.index + 1
so.head(9)
# -

# We know that the median popularity score is 38. Let's see where other popularities fall on the scale.

# +
#how many songs equal the median popularity?
len(so[so['popularity'] == 38])

#What does this mean
# -

# What is the most frequent popularity score?

so[['popularity']].mode()

len(so[so['popularity'] == 0])

#Where do songs with a 'middle' popularity score of 50 fall on the scale?
#There appears to be about 16k songs rated higher than 50
so[so['popularity'] == 50]

so[so['popularity'] >= 90]

so['popularity'].plot(kind='hist', bins=50)

# +
#find z-score for high popularity
pop = so['popularity']

z_of_90 = (90 - pop.mean()) / pop.std()
z_of_90
# -

low_pop = so[so['popularity'] <= 10]
len(low_pop)

#percentage of songs with popularity at or below 10
(len(low_pop) / len(so)) * 100

# The last few outputs demonstrate how rare songs are with high popularity ratings, and that a large portion of songs -- over 30% -- are virtually irrelevant. In fact, a score of 0 is by far the most frequent. 
#
# This illustrates the problem that popularity may not have a formula. I need to find a way to come to terms with the fact that a large part of the equation comes down to marketing and cultural factors. It might be beneficial to drop all rows with a popularity rating of 0 because, even if they have the exact same characteristics as popular songs, they may simply not be popular due to factors outside the scope of this analysis.

# #### Exploring and visualizing correlations

#which artists occur the most?
so[['track_name', 'artist_name']].groupby('artist_name').count().sort_values('track_name', ascending=False)

# Just some plot experiments below:

# +
so[so['popularity'] > 25].plot(kind='scatter', x='danceability', y='popularity', ylim=[15,100])

plt.show()

# +
so[so['popularity'] > 50].plot(kind='scatter', x='energy', y='popularity', ylim=[40,100], s=2)

plt.show()

# +
import seaborn as sns

sns.lmplot(x='energy', y='popularity', hue='mode', palette="Accent", scatter_kws={"s": 1}, data=so)
plt.show()

# +
above_70 = so[so['popularity'] > 70]

#compare with instrumentalness
sns.lmplot(x='instrumentalness', y='popularity', data=above_70, scatter_kws={"s": 10})
# -

#compare with liveness
sns.lmplot(x='liveness', y='popularity', data=above_70, scatter_kws={"s": 10})

#compare with speechiness
sns.lmplot(x='speechiness', y='popularity', data=so, scatter_kws={"s": 1})

#Attempt to make subplots: tempo vs. danceability
fig, axs = plt.subplots(ncols=2, sharex=False)
sns.regplot(x='tempo', y='popularity', data=above_70, scatter_kws={"s": 5}, ax=axs[0])
x = sns.regplot(x='danceability', y='popularity', data=above_70, scatter_kws={"s": 5}, ax=axs[1])
x.set(xlim=(0,1), ylim=(69,102))

# Attemp at feature selection:

# +
from sklearn.feature_selection import SelectKBest, chi2
    
X_reg = so.iloc[:, [2,3,4,5,6,11,12]]
y_reg = so.loc[:, 'popularity']
X_new = SelectKBest(chi2, k=5).fit_transform(X_reg, y_reg)

pd.DataFrame(X_new)
# -

# #### Missing values

so.info()

# There appears to no null values, but for the sake of practice a few more tests are below:

so.all()

so.any()

so[so.isnull()].count()









# Now let's do some basic exploration of the dataset

#What is the shape?
spotify.shape

#Examine the first few rows
spotify.head()

#Examine the tail
spotify.tail()

#gather some basic statistical info
spotify.describe()

#Narrow the focus to a couple key columns
spotify[['danceability', 'popularity']].describe()

# We can see from above that, based on the mean and upper quartile range, the large majority of songs listed in the dataset have a low popularity rating. Considering that popular songs are often characterized as "Top 40" or "Billboard 100", this shouldn't really be a surprise.
#

# Now let's a look at the metadata

spotify.info()


# Now we can sort the columns into two lists

#Define a sorting function
def col_types(df):
    """A function to sort columns into two lists based on whether they are
    numeric or non-numeric"""
    num=[]
    non_num=[]
    for col in df:
        if np.dtype(df[col]) in (['int32', 'int64', 'float32', 'float64', 'complex64', 'long']):
            num.append(df.loc[:, col].name)
        else:
            non_num.append(df.loc[:, col].name)
    print('These columns are numeric: ' + str(num)) 
    print( '\nThese columns are non-numeric: ' + str(non_num))


#run the function on the dataset
col_types(spotify)

# ## <b>json exercise playpen</b>

# +
projects = pd.read_json('data/world_bank_projects.json')

projects.columns

# +
projects = projects[['_id', 'approvalfy', 'country_namecode', 'countrycode', 'countryname', 'countryshortname', 'mjtheme', 'mjtheme_namecode', 'project_name']]

projects.shape

# +
grouped = projects[['countryshortname', 'project_name']].groupby('countryshortname').count()

#grouped.iloc[:10,:].sort_values('project_name', ascending=False)

ordered = grouped.sort_values('project_name', ascending=False)

top_10 = ordered.iloc[:10, :]

top_10
# -

# ## <b>question 2</b> 

projects.loc[491, 'mjtheme_namecode']

# +
themes = projects['mjtheme_namecode']

x = themes[1]

themes.head()

# +
counter = {}

for row in themes:
    for d in row:
        if d['code'] in counter:
            counter[d['code']] += 1
        else:
            counter[d['code']] = 1
            
counter

# +
import collections

count = collections.Counter()

for row in themes:
    for d in row:
        count[d['code']] += 1
        
count.most_common(10)

# +
import json
from pandas.io.json import json_normalize

x = pd.DataFrame()
#for row in themes:
#    x.append(json_normalize(row[0]))
    
x

# +
data = json.load(open('data/world_bank_projects.json'))

table = json_normalize(data, 'mjtheme_namecode', ['id'])

table.head(3)
# -

table[['name', 'code']].groupby('name').count().sort_values('code', ascending=False).head(10)

table[['name', 'code']].groupby('code').count().sort_values('name', ascending=False).head(10)



# ### Question 3

table.head(10)

# +
codes = list(table['code'])
names = list(table['name'])

zipped = list(zip(codes, names))
zipped = sorted(list(set(zipped)))

for tuple in zipped:
    if tuple[1] == '':
        zipped.remove(tuple)
    
zipped
# -

d = dict(zipped)
d

# +
for entry in themes:
    for theme in entry:
        if theme['name'] == '':
            theme['name'] = d[theme['code']]
            
type(themes)
# -

projects['mjtheme_namecode'] = themes
projects.loc[0, 'mjtheme_namecode']

StarcoderdataJupyterScriptsDedupFiltered