import arcgis
from getpass import getpass
import pandas as pd

"""
Explanation: Note: Increased IOPub
For visualization, if you did not start this notebook with an increased data rate limit, stop the notebook, go back to the command line, and start Jupyter Notebook using the following command.
jupyter notebook --NotebookApp.iopub_data_rate_limit=10000000000
Import the Python API module and Instantiate the GIS object
Import the Python API
End of explanation
"""


# gis_retail = arcgis.gis.GIS('Pro')  # optional, use if can use ArcGIS Pro authentication
gis = arcgis.gis.GIS(
    username='jmccune_retail', 
    password=getpass('Please enter the headless password: ')
)

"""
Explanation: Create an GIS object instance using the account currently logged in through ArcGIS Pro
End of explanation
"""


trade_area_itemid = '44c0b666404c41b8a14e04c34d09be1e'
item = arcgis.gis.Item(gis=gis, itemid=trade_area_itemid)
item

"""
Explanation: Get a Data Frame, data to work with, from the Web GIS Item ID
Create a Web GIS Item instance using the Item ID
End of explanation
"""


feature_layer = item.layers[0]
feature_layer

"""
Explanation: Since the item only contains one feature layer, get the first layer in the item, the Feature Layer we need to work with.
End of explanation
"""


feature_set = feature_layer.query()

df_fields = pd.DataFrame([(field['name'], field['alias']) for field in feature_set.fields], columns=['NAME', 'ALIAS'])
df_fields

"""
Explanation: Use query to return the data as a Feature Set.
End of explanation
"""


df = feature_set.df
df.head()

"""
Explanation: Take advantage of the df function on the Feature set object returned from the query to convert the data to a Pandas Data Frame.
End of explanation
"""


df.columns = [field.replace('Target_Locations_', '').replace('overlayTemplate_', '') for field in df.columns]
df.head()

"""
Explanation: Because I am neurotic, and hate the prefixes added to the field names from the join when preparing the data in ArcMap, we are cleaning them up a little here.
End of explanation
"""


from sklearn.cluster import KMeans
import pandas as pd
import numpy as np

%matplotlib inline

"""
Explanation: Store Segmentation Using KMeans Clustering
Use KMeans cluster analysis from the Sci-Kit Learn package to segment the demographically similar stores together.
End of explanation
"""


km = KMeans(n_clusters=5)

"""
Explanation: Create the KMeans clusterer, specifying it to create five clusters, or segments.
End of explanation
"""


df_independent = df.select_dtypes(include=[np.number]).drop(['OBJECTID', 'SALESVOL'], axis=1)
fields_independent = df_independent.columns  # save for later
field_name_dependent = 'SALESVOL' # keep track of this as well

df['segment'] = km.fit_predict(df_independent)

df.head()

"""
Explanation: Use the fit_predict method to create the clusters using just the numeric fields - not including the OID and sales fields.
End of explanation
"""


df_count = df.segment.value_counts().reset_index().rename(columns={'index': 'segment', 'segment': 'count'})
df_mean = df.groupby('segment').mean()[field_name_dependent].reset_index().rename(
    columns={'segment': 'segment', field_name_dependent: 'mean_sales'})
df_median = df.groupby('segment').median()[field_name_dependent].reset_index().rename(
    columns={'segment': 'segment', field_name_dependent: 'median_sales'})

df_summary = pd.merge(df_count, df_mean)
df_summary = pd.merge(df_summary, df_median)
df_summary = df_summary.sort_values('mean_sales', ascending=False)
df_summary['mean_sales_zscore'] = df_summary.apply(lambda row: (row.mean_sales - df.SALESVOL.mean()) / df.SALESVOL.std(ddof=0), axis=1)
print('Sales Mean: {:.2f}\nSales Standard Deviation: {:.2f}'.format(df.SALESVOL.mean(), df.SALESVOL.std()))
df_summary

"""
Explanation: Investigate the Results
Now, take a look at a few descriptive metrics to begin to understand the results - initially by creating a data frame with the segments, mean sales for each segment, median sales for each segment, and the median sales for each segment.
End of explanation
"""


ax = df_summary.sort_values('mean_sales').plot.bar(y='mean_sales', x='segment', figsize=(15, 8))
ax.set_ylabel('mean sales')
ax.set_title('Mean Sales by KMeans Segment')

ax.axhline(y=df.SALESVOL.mean(), color='r') # add the mean for reference
ax.axhline(y=df.SALESVOL.mean() - df.SALESVOL.std() * 0.5, color='g')
ax.axhline(y=df.SALESVOL.mean() + df.SALESVOL.std() * 0.5, color='g')

df['segment_3'] = df.segment == 3
df

df.groupby("segment_3").CITY.value_counts()

"""
Explanation: Now, using data from this data frame, visualize the means of each segment against the mean, and one-half standard deviation above and below the mean.
End of explanation
"""


top_segment = df_summary.iloc[0].segment
top_segment

"""
Explanation: Map Results
After doing this analysis, we are going to begin by showing the locations of the top performing stores on the map relative to the rest of the stores.
End of explanation
"""


store_map = gis.map('Meades Ranch, KS', 4)
store_map.basemap = 'gray'
store_map

for index, row in df[df.segment != top_segment].iterrows():
    store_map.draw(row.SHAPE, symbol={'type': 'esriSMS', 'style': 'esriSMSCircle', 'size': 4})
for index, row in df[df.segment == top_segment].iterrows():
    store_map.draw(row.SHAPE, symbol={'type': 'esriSMS', 'style': 'esriSMSCircle', "color":[0,255,0,255], "size":6})

"""
Explanation: This is a first stab at mapping the data just using the draw method, which plots the data on the map much like just using a sharpie. The data is not organized into a layer per say, so it is not very useful. Still, it is interesting at a coursory level, just seeing the rest of the stores, and the top performers.
End of explanation
"""


from sklearn.decomposition import PCA
import matplotlib.pyplot as plt

pca = PCA(n_components=2)

df['x'] = pca.fit_transform(df[fields_independent])[:,0]
df['y'] = pca.fit_transform(df[fields_independent])[:,1]

ax = plt.scatter(data=df, x='x', y='y', c='segment')

cluster_centers = pca.transform(cluster.cluster_centers_)
cluster_centers = pd.DataFrame(cluster_centers, columns=['x', 'y'])
cluster_centers['segment'] = range(0, len(cluster_centers))

plt.plot(data=cluster_centers)

plt.show()

"""
Explanation: PCA
End of explanation
"""


