X = X[X['lon'] < -122]
X.plot(kind='scatter', x='lon', y='lat')

"""
Explanation: As seen above we have a clear outsider that lies way outside SF, probably a typo. Hence we sort this datapoint out,
End of explanation
"""


from sklearn.cluster import KMeans

#To work with out cluster we have to turn our panda dataframe into a numpy array,
np_X = np.array(X)

kmeans = KMeans(n_clusters=2)
kmeans.fit(np_X)

centroid = kmeans.cluster_centers_
labels = kmeans.labels_

print "The %s cluster centers are located at %s " %(len(centroid),centroid)

colors = ["g.","r.","c."]

for i in range(len(np_X)):
   plt.plot(np_X[i][0],np_X[i][1],colors[labels[i]],markersize=10)

plt.scatter(centroid[:,0],centroid[:,1], marker = "x", s=150, linewidths = 5, zorder =10)

plt.show()

"""
Explanation: The points now all seem to be within SF borders
End of explanation
"""


from sklearn.cluster import KMeans

#To work with out cluster we have to turn our panda dataframe into a numpy array,
np_X = X

kmeans = KMeans(n_clusters=2)
kmeans.fit(np_X)

centroid = kmeans.cluster_centers_
classified_data = kmeans.labels_
labels = kmeans.labels_

print "The %s cluster centers are located at %s " %(len(centroid),centroid)

classified_data

#copy dataframe (may be memory intensive but just for illustration)
df_processed = X.copy()
df_processed['Cluster Class'] = pd.Series(classified_data, index=df_processed.index)

df_processed.head()

centroid_df = DataFrame(centroid)

centroid_df.head()

df_processed.plot(kind='scatter', x='lon', y='lat',
                  c = 'Cluster Class', label='datapoints');

"""
import numpy 
import pandas
from  matplotlib import pyplot
import seaborn
seaborn.set(style='ticks')

numpy.random.seed(0)
N = 37
_genders= ['Female', 'Male', 'Non-binary', 'No Response']
df = pandas.DataFrame({
    'Height (cm)': numpy.random.uniform(low=130, high=200, size=N),
    'Weight (kg)': numpy.random.uniform(low=30, high=100, size=N),
    'Gender': numpy.random.choice(_genders, size=N)
})

fg = seaborn.FacetGrid(data=df, hue='Gender', hue_order=_genders, aspect=1.61)
fg.map(pyplot.scatter, 'Weight (kg)', 'Height (cm)').add_legend()

########################################

import seaborn
seaborn.set(style='ticks')

fg = seaborn.FacetGrid(data=df_processed, hue='Cluster Class', hue_order=_classes, aspect=1.61)
fg.map(pyplot.scatter, 'Lat', 'Lon').add_legend()

"""

from scipy.spatial import distance

def dist_euc(lon,lat,centroid):
    data_cord = [lon,lat]
    return distance.euclidean(data_cord,centroid)

df_processed['distance'] = df_processed.apply(lambda row: dist_euc(row['lon'], row['lat'],centroid[row['Cluster Class']]), axis=1)

df_processed.head()

ksum = []
def get_ksum(k):
    lonList = X['lon'].tolist()
    latList = X['lat'].tolist()
    
    for i in range(1,k):
        kmeans = KMeans(n_clusters=i)
        kmeans.fit(X)
        centroid = kmeans.cluster_centers_
        labels = kmeans.labels_
        
        tmp_sum = 0
        for index, row in enumerate(lonList):
            tmp_sum += dist_euc(lonList[index], latList[index], centroid[labels[index]])
        ksum.append(tmp_sum)
    
get_ksum(10)

print ksum

#I Transform my data into a Dataframe to do easy and pretty plotting :-) 
ksum_df = DataFrame(ksum, index = range(1,10))

ksum_df.plot()

"""
Explanation: I will now look at the total squared error in relation to the number of clusters, to find the ideal knee bend,
End of explanation
"""


import csv
csv_file = df_processed[['lon','lat','Cluster Class']].values

csv_file

with open('datapoints.csv','wb') as f:
    w = csv.writer(f)
    w.writerows(csv_file)


df_csv.head()

df_csv = X.copy(deep = True)
centroid_list = []
for i in range(1,7):
    kmeans = KMeans(n_clusters=i)
    kmeans.fit(X)
    centroid = kmeans.cluster_centers_
    labels = kmeans.labels_
    column = "k%s" %i
    df_csv[column] = labels
    centroid_not_np = centroid.tolist()
    centroid_list.append(centroid_not_np)

df_csv.head()

centroid_list

df_csv.to_csv('csv_clusters.csv', index=False)

with open('centroids.csv','wb') as csvfile:
    w = csv.writer(csvfile,quoting=csv.QUOTE_MINIMAL)
    w.writerows(centroid_list)

"""
Explanation: As seen the error drops dramaticly as we move from 1 to 2 clusters. It also drops rather significantly from 2-3, though
not anything as much as the prior. The optimal solution would hence be either 2 or 3 clusters.
CSV exporter for D3 data
End of explanation
"""


