# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

#Import all reuired libraries
import pandas as pd
import datetime
import geopandas as gpd
from shapely.geometry import Point
from datetime import datetime
from datetime import timedelta
from sklearn import preprocessing
import category_encoders as ce 
from sklearn.preprocessing import StandardScaler
from sklearn.cluster import KMeans
import matplotlib.pyplot as plt
import numpy as np
from yellowbrick.cluster import KElbowVisualizer
from sklearn.metrics import silhouette_samples, silhouette_score
import matplotlib.cm as cm
import os


# The primary dataset which will be used in this analysis is the victim based part 1 crime table, produced by the Baltimore City Police Department. It reports crimes based upon each victim, rather than occurance, and thus each row can be thought of as a victim. 

#Crime = pd.read_csv('https://data.baltimorecity.gov/api/views/wsfq-mvij/rows.csv?accessType=DOWNLOAD')
Crime = pd.read_csv('C:/Users/cyeager/Desktop/UMBC/DATA 602/Homework2/Crime.csv',sep=",")

print(Crime.columns)

#Keep only the columns we want
df=Crime.filter(['CrimeDate', 'CrimeTime','Description','Latitude','Longitude'])

#Concat the date and time columns into a single feature
df['DT']=df['CrimeDate'] + ' ' + df['CrimeTime']
#df

#reformat into a new datetime column
df['DT']=pd.to_datetime(df['DT'])
#df

# Here we zip the latitude and longitude features into one, then apply the point function so that they can be displayed as a single dot of a map for each crime.

df['Coordinates'] = list(zip(df['Longitude'], df['Latitude']))
df['Coordinates'] = df['Coordinates'].apply(Point)

#All data prior to Jan 1 2014 was found to be sparse and inconsistent and thus is removed
df=df[(df['DT'] >= '2014-01-01 00:00:01')]
df['DT'].min()

#Save of a copy of this to come back to if we mess it up
df_saved=df

#Save of a copy of this to come back to if we mess it up
df_saved.to_pickle('CrimemapData.pkl')

df = pd.read_pickle('CrimemapData.pkl')

#Remove na values, only a few are lost in proportion to overall size, not worth investigating to potentially fix or interpolate
print(df.shape)
df=df.dropna(axis=0,how='any')
df=df.dropna(axis=1,how='any')
print(df.shape)

# We are going to break this data down into 2 hours intervals of time where we look at all of the crime which is taking place in the city during that snapshot and see if we can find a pattern that acts as a precursor to homicides/shootings. The first thing we do here is figure out how many 2 hours groups we can get from this dataset:

print(df['DT'].min())
print(df['DT'].max())
T=(df['DT'].max())-(df['DT'].min())

#here we find the total number of hours contained within our df and then see how many 2 hour chunks there are within in.
#We can create 29951 images 
days, seconds = T.days, T.seconds
hours = days * 24 + seconds // 3600
hours/2

#We want each type of crime to be a specific color
ClusterColors = ['#E47A2E', '#3F69AA', '#D5AE41', '#BD3D3A','#BD3D3A',"#006E6D",'#7F4145','#223A5E','#BD3D3A', '#3F69AA', '#D5AE41', '#766F57','#E47A2E',"#006E6D",'#7F4145','#223A5E',"#9C9A40","#9C9A40",'blue','green','red','purple','brown','#BD3D3A', '#3F69AA', '#D5AE41', '#766F57','#E47A2E',"#006E6D",'#7F4145','#223A5E',"#9C9A40"]

#Here we assign the color and size to each crime type point
CrimeColours = list(zip(list(df['Description'].unique()), ClusterColors))
CrimeColours=pd.DataFrame(CrimeColours)
CrimeColours['shape']=50

#Rename Colmns
CrimeColours.columns=['crime','color','shape']

df = pd.merge(df, CrimeColours, left_on=["Description"], right_on=['crime'])

# Running the loop to create almost 30k maps is very time consuming. Via trial and error it was found that the df had a few flaws which were not immediately noticeable. One of these was that a hadful of points actually fell outside the city due to geocoding errors. Here we find those points via spatially joining all of the data to a polygon of the city, then filtering the main df of those without a match.

Outsidezbmore=gpd.sjoin(BaltimoreCommunities,gpd.GeoDataFrame(df, geometry='Coordinates').set_crs("EPSG:4326"), op='contains')

Outsidezbmore.index_right
main_list = np.setdiff1d(list(df.index),list(Outsidezbmore.index_right))
main_list

print(df.shape)
df=df.drop(df.index[main_list])
print(df.shape)

#save it again
df.to_pickle('CrimemapDataClensed.pkl')

df = pd.read_pickle('CrimemapDataClensed.pkl')
df.head()

# Finally, the loop below creates a new df for each 2 hour window within the master dataframe, then appends each of these new, smaller dfs to a list.

maybe=[]
X=df['DT'].min()
Y=df['DT'].min()- timedelta(hours=2)
for i in range(29950):
    X=X + timedelta(hours=2)
    Y=Y + timedelta(hours=2)
    maybe.append(df[(df.DT >=Y) & (df.DT <=X)])
        
    #print(str(Y)+" - "+str(X))

# We create a list of the 2 hour dfs which contain either homicide or shooting for identification purposes since this is a classification model. Using this data we will place each image into a folder that fits its classification (homicide/shooting or none).

searchfor=["SHOOTING","HOMICIDE"]
um=[]
for i in range(len(maybe)):
    
    um.append(maybe[i][maybe[i]['Description'].str.contains('|'.join(searchfor))].empty)

#Read in the neighborhoods boundary shapefile from API
BaltimoreCommunities = gpd.read_file('https://data.baltimorecity.gov/api/geospatial/peh2-3qyi?method=export&format=Shapefile')

len(maybe)

# The final data issue to overcome was filtering those 2 hour windows in which no crime at all took place. These would cause an error as the mapping operation is disrupted by empty dfs. Rather than deleted them, I simply incorporated an if statement with 'continue' line into the main mapping loop.

dud=[]
for i in range(len(maybe)):
    dud.append(maybe[i].empty)

skiplist=[i for i,x in enumerate(dud) if x == True]
#skiplist

#Only 137 of the nearly 30k 2 hour windows contained no crime.
len(skiplist)

# This loop creates one map image for each 2 hour window. A point is displayed for each crime, it has a specific color which corresponds to its type. I've also spatially joined crime to the neighborhood in which it belongs and created a red outline for each neighbrhood where there was crime, all others are outlined in black. Once the map is created its given a unique name and saved as a .png file into a folder which dsignates its status as having a homicide or not. 

for i in range(0,len(maybe)):
    if maybe[i].empty==True:
        continue    
    base = BaltimoreCommunities.plot(color='none', edgecolor='black',alpha=.5)

    base.set_aspect('equal')
    base.set_axis_off()

    ax=gpd.GeoDataFrame(maybe[i], geometry='Coordinates').set_crs("EPSG:4326").plot(ax=base,c=maybe[i]['color'], markersize = maybe[i]['shape'],alpha=.7)

    gpd.sjoin(BaltimoreCommunities,gpd.GeoDataFrame(maybe[i], geometry='Coordinates').set_crs("EPSG:4326"), op='contains').plot(ax=base,edgecolor='red',color='none')
  
    fig = ax.figure


    File=(os.getcwd()+"\Maps"+"\\"+str(um[i])+"\\"+maybe[i].DT.max().strftime(format='%Y%m_%I%M')+str(i)+".png")
    fig.savefig(File, bbox_inches = 'tight',pad_inches = 0)
    plt.close('all')

# Out of our 29951 images, 'False' designates that there was a homicide or shooting during that 2 hour window. There were 4824 of these. The remaining 24989 (designated 'True') were windows of time without a shooting or homicide. 



StarcoderdataJupyterScriptsDedupFiltered