# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3.6
#     language: python
#     name: python3
# ---

# <h1>SidLabs_Segmenting Neighborhoods in Torronto</h1>
# <h3>Question#1</h3>
# <h4>Using pandas to transform the data in the provided table from Wikipedia page into pandas dataframe.</h4>
# <p>1.1 Importing libraries and solving environments<p>

# +
import numpy as np # importing numpy
import pandas as pd # importing pandas
pd.set_option('display.max_columns', None)
pd.set_option('display.max_rows', None)
import json # importing JSON
# #!conda install -c conda-forge geopy --yes 
from geopy.geocoders import Nominatim # convert to latitude and longitude
import requests # importing requests
from pandas.io.json import json_normalize # JSON into dataframe
import matplotlib.cm as cm # importing matplotlib
import matplotlib.colors as colors

from sklearn.cluster import KMeans # importing K-Means
from sklearn.datasets.samples_generator import make_blobs
# #!conda install -c conda-forge folium=0.5.0 --yes
import folium # importing folium
from bs4 import BeautifulSoup
import lxml
print("all done here!")
# -

# <p>1.2  Scraping and downloading the data using BeautifulSoup and loading it into Pandas dataframe.</p>

# +
r = requests.get('https://en.wikipedia.org/wiki/List_of_postal_codes_of_Canada:_M')
soup = BeautifulSoup(r.text, 'html.parser')
table=soup.find('table', attrs={'class':'wikitable sortable'})

#heads...
headers=table.findAll('th')
for i, head in enumerate(headers): headers[i]=str(headers[i]).replace("<th>","").replace("</th>","").replace("\n","")
rows=table.findAll('tr')
rows=rows[1:len(rows)]

# skipping symbols and line feeds...
for i, row in enumerate(rows): rows[i] = str(rows[i]).replace("\n</td></tr>","").replace("<tr>\n<td>","")

df=pd.DataFrame(rows)
df[headers] = df[0].str.split("</td>\n<td>", n = 2, expand = True) 
df.drop(columns=[0],inplace=True)

df = df.drop(df[(df.Borough == "Not assigned")].index)
df.Neighbourhood.replace("Not assigned", df.Borough, inplace=True)
df.Neighbourhood.fillna(df.Borough, inplace=True)
df=df.drop_duplicates()

# extract titles...
df.update(
    df.Neighbourhood.loc[
        lambda x: x.str.contains('title')
    ].str.extract('title=\"([^\"]*)',expand=False))

df.update(
    df.Borough.loc[
        lambda x: x.str.contains('title')
    ].str.extract('title=\"([^\"]*)',expand=False))

df.update(
    df.Neighbourhood.loc[
        lambda x: x.str.contains('Toronto')
    ].str.replace(", Toronto",""))
df.update(
    df.Neighbourhood.loc[
        lambda x: x.str.contains('Toronto')
    ].str.replace("\(Toronto\)",""))

# combining neighborhoods with the same post code...
df2 = pd.DataFrame({'Postcode':df.Postcode.unique()})
df2['Borough']=pd.DataFrame(list(set(df['Borough'].loc[df['Postcode'] == x['Postcode']])) for i, x in df2.iterrows())
df2['Neighborhood']=pd.Series(list(set(df['Neighbourhood'].loc[df['Postcode'] == x['Postcode']])) for i, x in df2.iterrows())
df2['Neighborhood']=df2['Neighborhood'].apply(lambda x: ', '.join(x))
df2.dtypes
# -

# <p>1.2.1 having a look at our dataframe</p>

df2.head(10)

# <p>1.3 using the .shape method to print the number of rows of the dataframe.</p>

df2.shape

# <h3>Question#2</h3>
# <h4>Using the Geocoder package or the csv file to create dataframe with longitude and latitude values.</h4>
# <p>2.1 Downloading the toronto longitude and latitude data from cocl.us site and exploring the dataframe.

# #!wget -q -O 'Toronto_long_lat_data.csv'  http://cocl.us/Geospatial_data
df_lon_lat = pd.read_csv('Toronto_long_lat_data.csv')
df_lon_lat.head()

# <p> 2.1.1 Converting the column name 'Postal Code" on df_lon_lat to match "Postcode" from df2.</p>

df_lon_lat.columns=['Postcode','Latitude','Longitude']
df_lon_lat.head()

# <p> 2.2 merging df2 and df_lon_lat into df3. </p>

df3 = pd.merge(df2,
                 df_lon_lat[['Postcode','Latitude', 'Longitude']],
                 on='Postcode')
df3.head(15)

# <h3>Question#3</h3>
# <p>
#    Exploring and clustering the neighborhoods in Toronto. 
#    Also use geopy library to get the latitude and longitude values for Toronto.
# </p>
# <p> 3.1 Importing libraries and solving environments </p>

from geopy.geocoders import Nominatim
import matplotlib.cm as cm
import matplotlib.colors as colors
from sklearn.cluster import KMeans
# #!conda install -c conda-forge folium=0.5.0 --yes
import folium
print('All done, Plz proceed...')

# 3.2 Using the geolocator library to obtain the longitude and latitude values for Toronto</p>

address = 'Toronto, ON'
geolocator = Nominatim(user_agent="Toronto")
location = geolocator.geocode(address)
latitude_toronto = location.latitude
longitude_toronto = location.longitude
print('The coordinates for Toronto are {}, {}.'.format(latitude_toronto, longitude_toronto))

# <p>3.2 plotting the neighborhoods on the Toronto map to visualize</p>

# +
map_toronto = folium.Map(location=[latitude_toronto, longitude_toronto], zoom_start=10)

# adding markers to map ...
for lat, lng, borough, Neighborhood in zip(df3['Latitude'], df3['Longitude'], df3['Borough'], df3['Neighborhood']):
    label = '{}, {}'.format(Neighborhood, borough)
    label = folium.Popup(label, parse_html=True)
    folium.CircleMarker(
        [lat, lng],
        radius=5,
        popup=label,
        color='red',
        fill=True,
        fill_color='green',
        fill_opacity=0.7,
        parse_html=False).add_to(map_toronto)  
    
map_toronto
# -



StarcoderdataJupyterScriptsDedupFiltered