# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# ### Creating the main table for part 3 (from part 1 and 2)

# +
#import the libraries BeautifulSoup and io and pandas
import bs4 as bs
from bs4 import BeautifulSoup
import io
import urllib.request
import pandas as pd

#importing more libraries
# Matplotlib and associated plotting modules
import matplotlib.cm as cm
import matplotlib.colors as colors

# import k-means from clustering stage
from sklearn.cluster import KMeans

# Importing libraries and APIs to convert a neighbourhood and borough into latitude and longitude values and create dataframes
import numpy as np # library to handle data in a vectorized manner
import pandas as pd # library for data analsysis
pd.set_option('display.max_columns', None)
pd.set_option('display.max_rows', None)
import json # library for JSON files
# !conda install -c conda-forge geopy --yes 
from geopy.geocoders import Nominatim # to get latitude and longitude values
import requests # it handles requests
from pandas.io.json import json_normalize # from JSON file to a pandas dataframe

#Creating the source from which to scrape the data from the table called "List of postal codes of Canada: M" from thr wikipedia page
source = urllib.request.urlopen('https://en.wikipedia.org/wiki/List_of_postal_codes_of_Canada:_M').read()

#Creating table object with BeautifulSoup
soup=BeautifulSoup(source, 'lxml')
table=soup.table

#Scraping the columns' names from "List of postal codes of Canada: M" from thr wikipedia
table_header=table.find_all('th')
for th in table_header:
    headers=[]
    headers.append(th.text)

# Creating a table with pandas that contains the data scraped from "List of postal codes of Canada: M" from thr wikipedia
table_rows=table.find_all('tr')
list_of_rows=[]
for tr in table_rows:
    td=tr.find_all('td')
    row = [item.text.strip('\n') for item in td]
    list_of_rows.append(row)
    
df=pd.DataFrame(list_of_rows, columns=["Postcode", "Borough", "Neighbourhood"]) 
df=df.iloc[1:]

#Keeping the strings for the column 'Borough' that are different from 'Not assigned'
df=df[df['Borough'] != 'Not assigned']
df=df.iloc[:].reset_index(drop=True)

#The 'Not assigned' values in the column 'Neighbourhood' are overwritten with the corresponding string in the column 'Borough'
for i, row in df.iterrows():
    if df.loc[i,'Neighbourhood'] == 'Not assigned':
        df.loc[i,'Neighbourhood'] = df.loc[i, 'Borough'] 
        
#Rows of the dataframe are grouped by 'Postcode' and 'Postcode' and
# the corresponding 'Neighbourhoods are combined into one row with the neighborhoods separated with a comma 
df_1 = df.groupby(['Postcode', 'Borough'])['Neighbourhood'].apply(lambda x: x.str.cat(sep=', '))

#Table from part 1 that I am going to use in this notebook (part2)
df_final_part_1= pd.DataFrame(df_1).reset_index(drop=False)

#Getting the geospatial data (longitude, latitude and postcode) in a dataframe (table)
url="http://cocl.us/Geospatial_data"
source_toronto=requests.get(url).content
long_lat_postcode_df=pd.read_csv(io.StringIO(source_toronto.decode('utf-8')))

#Adding latitude and longitude to the dataframe obtained from part_1
Table_part_2 = pd.merge(df_final_part_1,
                        long_lat_postcode_df,
                        left_on='Postcode',
                        right_on='Postal Code',
                        how='left')

#Dropping the duplicate column 'postal code' (it's like Postcode)
#Final Table for Part 2 of the IBM project
Table_part_2.drop(['Postal Code'], axis=1)

# -

#To have an idea of the number of neighbourhoods and boroughs in Toronto.
print('Toronto has {} boroughs and {} neighbourhoods.'.format(
        len(Table_part_2['Borough'].unique()),
        len(Table_part_2['Neighbourhood'].unique())
         )
     )

#Toronto latitude and longitude using the geopy library
address = 'Toronto'
geolocator = Nominatim(user_agent="toronto_explorer")
location = geolocator.geocode(address)
latitude = location.latitude
longitude = location.longitude
print('The geograpical coordinate of Toronto are {}, {}.'.format(latitude, longitude))

# ### Visualizing neighborhoods on Toronto's map.

# +
# For Toronto I am using the same method used for New York and I also import Folium for map visualization of the 
# !pip install folium
import folium 

# create map of Toronto using latitude and longitude values
map_toronto = folium.Map(location=[latitude, longitude], zoom_start=10)

# add markers to map
for lat, lng, borough, neighborhood in zip(Table_part_2['Latitude'], Table_part_2['Longitude'], Table_part_2['Borough'], Table_part_2['Neighbourhood']):
    label = '{}, {}'.format(neighborhood, borough)
    label = folium.Popup(label, parse_html=True)
    folium.CircleMarker(
        [lat, lng],
        radius=5,
        popup=label,
        color='yellow',
        fill=True,
        fill_color='green',
        fill_opacity=0.7,
        parse_html=False).add_to(map_toronto)  
    
map_toronto
# -

# ###  Exploring Scarborough borough in Toronto with Foursquare API and segment them.

# +
CLIENT_ID = 'CZY<KEY>' # my Foursquare ID
CLIENT_SECRET = '<KEY>' # my Foursquare Secret
VERSION = '20180605' # Foursquare API version

print('Your credentails:')
print('CLIENT_ID: ' + CLIENT_ID)
print('CLIENT_SECRET:' + CLIENT_SECRET)
# -

Table_part_2.loc[0, 'Borough'] #exploring the 1st borough of the dataframe Table_part_2

scarborough_df = Table_part_2[Table_part_2['Borough'] == 'Scarborough'].reset_index(drop=True)
scarborough_df.head(2)

# +
borough_latitude = scarborough_df.loc[0, 'Latitude'] # neighborhood latitude value
borough_longitude = scarborough_df.loc[0, 'Longitude'] # neighborhood longitude value

borough_name = scarborough_df.loc[0, 'Borough'] # neighborhood name

print('Latitude and longitude values of {} are {}, {}.'.format(borough_name, 
                                                               borough_latitude, 
                                                            borough_longitude))
# -

# ### Creating a map of Scarborough

#Scarborough latitude and longitude using the geopy library
address = 'Scarborough, Toronto'
geolocator = Nominatim(user_agent="scarborough_explorer")
location = geolocator.geocode(address)
latitude = location.latitude
longitude = location.longitude
print('The geograpical coordinate of Scarborough are {}, {}.'.format(latitude, longitude))

address_scarbour = 'Scarborough'
latitude_scarbour = latitude
longitude_scarbour = longitude
print('The geograpical coordinate of Scarborough are {}, {}.'.format(latitude_scarbour, longitude_scarbour))

# +
# create map of Toronto using latitude and longitude values
map_scarbour = folium.Map(location=[latitude_scarbour, longitude_scarbour], zoom_start=10)

# add markers to map
for lat, lng, neighborhood in zip(scarborough_df['Latitude'], scarborough_df['Longitude'], scarborough_df['Neighbourhood']):
    label = '{}, {}'.format(neighborhood, borough)
    label = folium.Popup(label, parse_html=True)
    folium.CircleMarker(
        [lat, lng],
        radius=5,
        popup=label,
        color='yellow',
        fill=True,
        fill_color='green',
        fill_opacity=0.7,
        parse_html=False).add_to(map_scarbour)  
    
map_scarbour
# -

# #### Top 100 venues that are in Scarborough within a radius of 1000 meters

#setting 100 venues and 500 meters radius
LIMIT = 100 # limit of number of venues returned by Foursquare API
radius = 500 # define radius
# create URL
url = 'https://api.foursquare.com/v2/venues/explore?&client_id={}&client_secret={}&v={}&ll={},{}&radius={}&limit={}'.format(
    'CZY<KEY>', 
    'KBY2IB4DOTCOS2BJNT32P1WX2OXKNAO2WO3VVHUWHRDUIUMX', 
    '20180605' , 
    latitude_scarbour, 
    longitude_scarbour, 
    radius, 
    LIMIT)
url # display URL

print(" Scarborough_latitude is: {},  Scarborough_longitude is: {}".format(latitude_scarbour, longitude_scarbour))

#Examining the results of the GET request 
results = requests.get(url).json()
results


# From Foursquare API function that extracts the category of the venue
def get_category_type(row):
    try:
        categories_list = row['categories']
    except:
        categories_list = row['venue.categories']
        
    if len(categories_list) == 0:
        return None
    else:
        return categories_list[0]['name']


# ### Top 100 venues in Scarborough

# +

venues = results['response']['groups'][0]['items']
    
nearby_venues = json_normalize(venues) # flatten JSON

# filter columns
filtered_columns = ['venue.name', 'venue.categories', 'venue.location.lat', 'venue.location.lng']
nearby_venues =nearby_venues.loc[:, filtered_columns]

# filter the category for each row
nearby_venues['venue.categories'] = nearby_venues.apply(get_category_type, axis=1)

# clean columns
nearby_venues.columns = [col.split(".")[-1] for col in nearby_venues.columns]

nearby_venues
# -

print('{} venues were returned by Foursquare.'.format(nearby_venues.shape[0]))


#Function to get the nearby venues (radius=500 meters, names:neighborhood) 
def getNearbyVenues(names, latitudes, longitudes, radius=500):
    
    venues_list=[]
    for name, lat, lng in zip(names, latitudes, longitudes):
        print(name)
            
        # create the API request URL
        url = 'https://api.foursquare.com/v2/venues/explore?&client_id={}&client_secret={}&v={}&ll={},{}&radius={}&limit={}'.format(
            CLIENT_ID, 
            CLIENT_SECRET, 
            VERSION, 
            lat, 
            lng, 
            radius, 
            LIMIT)
            
        # make the GET request
        results = requests.get(url).json()["response"]['groups'][0]['items']
        
        # return only relevant information for each nearby venue
        venues_list.append([(
            name, 
            lat, 
            lng, 
            v['venue']['name'], 
            v['venue']['location']['lat'], 
            v['venue']['location']['lng'],  
            v['venue']['categories'][0]['name']) for v in results])

    nearby_venues = pd.DataFrame([item for venue_list in venues_list for item in venue_list])
    nearby_venues.columns = ['Neighborhood', 
                  'Neighborhood Latitude', 
                  'Neighborhood Longitude', 
                  'Venue', 
                  'Venue Latitude', 
                  'Venue Longitude', 
                  'Venue Category']
    
    return(nearby_venues)


# ### Venues are in the Scarborough's neighborhoods

#Which venues are in the Scarborough's neighborhoods (radius = 500 m)? The list of neighborhood is given by the getNearbyVenues function
#applied to the scarborough_df dataframe
scarborough_nearby_venues = getNearbyVenues(names=scarborough_df['Neighbourhood'],
                                   latitudes=scarborough_df['Latitude'],
                                   longitudes=scarborough_df['Longitude']
                                           )

#Getting the evnues in Scarborough's nearby neighborhoods (radius 500 meters)
scarborough_nearby_venue_df=pd.DataFrame(scarborough_nearby_venues)
scarborough_nearby_venue_df

#Checking the size of the resulting dataframe
print(scarborough_nearby_venue_df.shape)

# ### Exploring the venues' dataset for neighborhoods nearby Scarborough

#Number of Venues for neighborhood
scarborough_nearby_venue_df.groupby('Neighborhood').count()

#Unique neighbourhoods in Scarborough
scarborough_nearby_venue_df['Neighborhood'].unique().tolist()


#Number of unique neighbourhoods in Scarborough
print("The number of neighborhood in Scarborough, borough in Toronto is: {}.".format(len(scarborough_nearby_venue_df['Neighborhood'].unique().tolist())))

#Unique venue category in Scarborough
scarborough_nearby_venue_df['Venue Category'].unique().tolist()

#Number of unique neighbourhoods in Scarborough
print("The number of unique venues in Scarborough's nearby neighborhoods (Toronto) is: {}.".format(len(scarborough_nearby_venue_df['Venue Category'].unique().tolist())))

#Number of venues' categories in Scarborough's neighborhoods
most_frequent_venue_cat_scarborough=scarborough_nearby_venue_df.groupby(['Venue Category']).count()
venue_count=scarborough_nearby_venue_df.groupby(['Venue Category']).count()
venue_count=venue_count['Neighborhood'].tolist()
most_frequent_venue_cat_scarborough['Venue_count']=venue_count
venues_scar_count=most_frequent_venue_cat_scarborough.drop(['Neighborhood','Neighborhood Latitude','Neighborhood Longitude','Venue', 'Venue Latitude', 'Venue Longitude'], axis=1)
venues_scar_count=venues_scar_count.reset_index(drop=False)
venues_scar_count.rename(columns={'Venue Category':'Category'}, inplace=True)
venues_scar_count

# +
# Visualising Venues category count in Scorborough in Toronto with a basic barplot. There are lots of food stores and bars!!
import matplotlib.pyplot as plt
import numpy as np

bars = pd.Series(venues_scar_count.Category)
height = pd.Series(venues_scar_count.Venue_count)
y_pos = np.arange(len(bars))
plt.bar(y_pos, height)
 
# If we have long labels, we cannot see it properly
names=venues_scar_count.Category.tolist()
plt.xticks(y_pos, names, rotation=90)
 
# Thus we have to give more margin:
plt.subplots_adjust(bottom=0.4)
 
# It's the same concept if you need more space for your titles
plt.title("Venues in Scorborough's neighborhoods, Toronto")
plt.subplots_adjust(top=0.7)


# -

# ### Analyzing Each Neighborhood in Scarborough

#Getting back to the scarborough's nearby neighborhoods venues dataset
scarborough_nearby_venue_df=pd.DataFrame(scarborough_nearby_venues)
scarborough_nearby_venue_df

# +
# one hot encoding
scarb_onehot = pd.get_dummies(scarborough_nearby_venue_df[['Venue']], prefix="", prefix_sep="")

# add neighborhood column back to dataframe
scarb_onehot['Neighborhood'] = scarborough_nearby_venue_df['Neighborhood'] 

# move neighborhood column to the first column
fixed_columns = [scarb_onehot.columns[-1]] + list(scarb_onehot.columns[:-1])
scarb_onehot = scarb_onehot[fixed_columns]

scarb_onehot.head()
# -

# examining the new dataframe scarb_onehot size
scarb_onehot.size

# ### Grouping rows by neighborhood and by taking the mean of the frequency of occurrence of each category

scarborough_grouped = scarb_onehot.groupby('Neighborhood').mean().reset_index()
scarborough_grouped

#checking the size
scarborough_grouped.size


# ### Ten top venues per neighborhood in Scarborough

def return_most_common_venues(row, num_top_venues):
    row_categories = row.iloc[1:]
    row_categories_sorted = row_categories.sort_values(ascending=False)
    
    return row_categories_sorted.index.values[0:num_top_venues]


# +
num_top_venues = 10

indicators = ['st', 'nd', 'rd']

# create columns according to number of top venues
columns = ['Neighborhood']
for ind in np.arange(num_top_venues):
    try:
        columns.append('{}{} Most Common Venue'.format(ind+1, indicators[ind]))
    except:
        columns.append('{}th Most Common Venue'.format(ind+1))

# create a new dataframe
neighborhoods_venues_sorted = pd.DataFrame(columns=columns)
neighborhoods_venues_sorted['Neighborhood'] = scarborough_grouped['Neighborhood']

for ind in np.arange(scarborough_grouped.shape[0]):
    neighborhoods_venues_sorted.iloc[ind, 1:] = return_most_common_venues(scarborough_grouped.iloc[ind, :], num_top_venues)

neighborhoods_venues_sorted
# -

# ### Cluster and label neighborhoods.
# ####  Here I set number of clusters equal to five and so the neighborhood will be labelled with an integers value (labels) ranging from 0 to 4 (included)

# +
# set number of clusters
kclusters = 5

scarborough_grouped_clustering = scarborough_grouped.drop('Neighborhood', 1)   #dropping the column 'Neighbourhood'

# run k-means clustering
kmeans = KMeans(n_clusters=kclusters, random_state=0).fit(scarborough_grouped_clustering)

# check cluster labels generated for each row in the dataframe
kmeans.labels_
print('kmeans_labels are the following array: {}, the number of labels are: {}'.format(kmeans.labels_, len(kmeans.labels_)))
# -

 scarborough_df  #This dataframe has 17 rows while the scarborough_grouped_clustering has only 16

neighborhoods_venues_sorted #This dataframe will ne joined to the scarborough_df but the number of rows is different. That's a problem.

#Checking whether 'Upper Rouge is present in 'Neighbourhood in the neighborhoods_venues_sorted dataframe. It's NOT.
(neighborhoods_venues_sorted['Neighborhood']=='Upper Rouge').any()  

# +
# 'Upper Rouge is NOT present in 'Neighbourhood' column in the neighborhoods_venues_sorted dataframe so I can remove it from
#scarborough_df and join the two datasets that have now the same numbero of rows.
scarborough_merged = scarborough_df.drop(16)

# add clustering labels to the scarborough_nearby_venue_df
scarborough_merged['Cluster Labels'] = kmeans.labels_

# merge toronto_grouped with toronto_data to add latitude/longitude for each neighborhood
scarborough_merged = scarborough_merged.join(neighborhoods_venues_sorted.set_index('Neighborhood'), on='Neighbourhood')

scarborough_merged
# -

#Getting the columns of the merged dataframe
scarborough_merged.columns

# ### Scarborough neighborhoods' five clusters visualised on Toronto's map 

# +
# create map
map_clusters = folium.Map(location = [latitude_scarbour, longitude_scarbour], zoom_start=11)

# set color scheme for the clusters
x = np.arange(kclusters)
ys = [i+x+(i*x)**2 for i in range(kclusters)]
colors_array = cm.rainbow(np.linspace(0, 1, len(ys)))
rainbow = [colors.rgb2hex(i) for i in colors_array]

# add markers to the map
markers_colors = []
for lat, lon, poi, cluster in zip(scarborough_merged['Latitude'], scarborough_merged['Longitude'], scarborough_merged['Neighbourhood'], scarborough_merged['Cluster Labels']):
    label = folium.Popup(str(poi) + ' Cluster ' + str(cluster), parse_html=True)
    folium.CircleMarker(
        [lat, lon],
        radius=5,
        popup=label,
        color=rainbow[cluster-1],
        fill=True,
        fill_color=rainbow[cluster-1],
        fill_opacity=0.7).add_to(map_clusters)
       
map_clusters
# -

# ### Checking 1st cluster labelled 0. 

scarborough_merged.loc[scarborough_merged['Cluster Labels'] == 0, scarborough_merged.columns[[1] + list(range(5, scarborough_merged.shape[1]))]]

# ## Checking 2nd cluster labelled 1

scarborough_merged.loc[scarborough_merged['Cluster Labels'] == 1, scarborough_merged.columns[[1] + list(range(5, scarborough_merged.shape[1]))]]

# ## Checking 3rd cluster labelled 2

scarborough_merged.loc[scarborough_merged['Cluster Labels'] == 2, scarborough_merged.columns[[1] + list(range(5, scarborough_merged.shape[1]))]]

# ## Checking 4th cluster labelled 3

scarborough_merged.loc[scarborough_merged['Cluster Labels'] == 3, scarborough_merged.columns[[1] + list(range(5, scarborough_merged.shape[1]))]]

# ## Checking 5th cluster labelled 4

scarborough_merged.loc[scarborough_merged['Cluster Labels'] == 4, scarborough_merged.columns[[1] + list(range(5, scarborough_merged.shape[1]))]]

# **Reference** 'Segmenting and Clustering Neighborhoods in New York City' by IBM



StarcoderdataJupyterScriptsDedupFiltered