# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3 (ipykernel)
#     language: python
#     name: python3
# ---

# # Run
#
# This notebook creates the neighbourhood aggregates from the preprocessed data.

# ## Settings

# + tags=[]
dist_folder = '../tmp/dist' # stores outputs
data_sources_file = 'data.csv'
study_area_dataset = 'toronto' # bbox will be used as study area
force = False # whether to force processing
genstats = ['sum', 'mean', 'min', 'max', 'median', 'var', 'skew', 'std', 'sem', 'mad'] # name of gen stats from pandas to agg numeric values
genstats_quantiles = True # whether to include quantiles in gen stats or not
# -

# ## Libraries

# + tags=[]
import pandas as pd
import geopandas as gpd
import numpy as np
import re

from datetime import datetime, timedelta
from IPython.display import display, HTML
from pathlib import Path
from shapely.geometry import box, Point
from time import time


# + [markdown] tags=[]
# ## Classes
# -

# For timing processes and displaying processing messages
class Timer:
    
    def __init__(self, name, process_name=None, start_prefix=None, end_prefix=None):
        self.name = name
        self.process_name = process_name
        self.start_prefix = start_prefix if start_prefix else 'Loading' if process_name.startswith('already') else 'Processing'
        self.end_prefix = end_prefix if end_prefix else 'Loaded' if process_name.startswith('already') else 'Processed'
    
    def start(self):
        self.start = time()
        self.start_dt = datetime.now()
        print(f'{self.start_prefix} {self.name} - {self.process_name} ({self.start_dt})...')
    
    def end(self):
        self.end = time()
        self.end_dt = datetime.now()
        self.elapsed = str(timedelta(seconds=self.end - self.start))
        print(f'{self.end_prefix} {self.name} - {self.process_name} ({self.end_dt}) [Elapsed: {self.elapsed}]')


# ## Functions

# ### Data Functions

# +
# Formats human readable sizes in bytes
# https://stackoverflow.com/questions/1094841/get-human-readable-version-of-file-size
def format_size(num, suffix="B"):
    for unit in ["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi"]:
        if abs(num) < 1024.0:
            return f"{num:3.1f} {unit}{suffix}"
        num /= 1024.0
    return f"{num:.1f} Yi{suffix}"

# Format cols to be lowercase with underscores and alphanumerics only
def format_cols(df):
    out = df
    out = out.rename(str.lower, axis='columns') # lowercase
    out.columns = [re.sub('[^A-Za-z0-9 ]+', '', c) for c in out.columns] # remove non alphanumeric
    out.columns = [' '.join(c.split()).replace(' ', '_') for c in out.columns] # remove extra whitespace, replace w/ underscore
    return out

# Extracts file info for summary
def file_info(file):
    stats = Path(file).stat()
    modified = datetime.fromtimestamp(stats.st_mtime)
    created = datetime.fromtimestamp(stats.st_birthtime)
    size = format_size(stats.st_size)
    out = {'created': created, 'modified': modified, 'size': size}
    return out

# Get basic info on a pandas dataframe
def df_info(df, name):
    memory = format_size(df.memory_usage().sum())
    cols = len(df.columns)
    rows = len(df)
    geometry = str(pd.unique(df.geometry.geom_type)) if 'geometry' in df else None
    out = {
        'dataset': name,
        'columns': cols,
        'rows': rows,
        'memory_usage': memory,
        'geometry': geometry
    }
    return out

# Read csv, shp, or geojson file into pandas or geopandas
def read_data(file, *args, **kwargs):
    path = Path(file)
    ext = path.suffix if path.suffix != '.zip' else path.suffixes[0]
    ext = ext.lower()
    if ext == '.csv':
        out = pd.read_csv(file, *args, **kwargs)
    elif ext in ['.geojson', '.shp']:
        out = gpd.read_file(file, *args, **kwargs)
    else:
        raise ValueError('File format not supported')
    return out

# Read processed dataset in dist folder
def read_clean_dataset(dataset, folder='../dist', *args, **kwargs):
    file = Path(f'{folder}/clean-datasets/{dataset}.geojson')
    out = read_data(file, *args, **kwargs)
    return out

# Check if dataset is processed in dist folder
def dataset_processed(dataset, folder='../dist'):
    file = Path(f'{folder}/clean-datasets/{dataset}.geojson')
    out = file.is_file()
    return out

# Save variable to csv files
def save_vars(gdf, dist_folder=dist_folder):
    for c in gdf.columns:
        if c not in ['neighbourhood_id', 'neighbourhood']:
            v = gdf[c]
            v.to_csv(f'{dist_folder}/by-variable/{c}.csv')


# -

# ### Aggregation Functions

# +
# Calculates the mode for a series
def mode(x):
    out = x.map(str).value_counts().index[0]
    return out

# 25 per quantile
def quantile25(df):
    out = df.quantile(0.25)
    return out

# 50 per quantile
def quantile50(df):
    out = df.quantile(0.5)
    return out

# 75 per quantile
def quantile75(df):
    out = df.quantile(0.75)
    return out

# Return a func for getting geom length aggs
def glen(agg):
    def out(x):
        return getattr(x.length, agg)()
    return out

# Calculates the counts of unique values given all possible values
def ucounts(df, c, possible):
    possible = [str(k) for k in possible]
    counts = df[c].value_counts().to_dict()
    out = {k:[0] for k in possible}
    for k, v in counts.items():
        out[str(k)] = [v]
    out = pd.DataFrame(out)
    return out


# -

# ### Metric Functions

# Calculate line sinuosity (how curved lines are)
def lsin(geom):
    
    # Calculate straight length
    start = gpd.GeoSeries([Point(g.coords[0]) for g in geom])
    end = gpd.GeoSeries([Point(g.coords[-1]) for g in geom])
    straight_len = start.distance(end).reset_index(drop=True)
    
    # Calculate sin = actual / straight
    actual_len = geom.length.reset_index(drop=True)
    out = actual_len / straight_len
    return out


# ## Run
#
# Create folders to store processed data:

# Create folders for clean and output data
Path(dist_folder).mkdir(parents=True, exist_ok=True)
for f in ['by-dataset', 'by-variable']:
    Path(f'{dist_folder}/{f}').mkdir(parents=True, exist_ok=True)

# Track processing time for all datasets and procedures:

# + tags=[]
gtimer = Timer('all datasets', 'preprocessing')
gtimer.start()
# -

# Include quantiles in general statistics for aggregating numeric values during spatial binning:

if genstats_quantiles:
    genstats = genstats + [quantile25, quantile50, quantile75]

# Load neighbourhood polygons:

nbhoods = read_data(f'{dist_folder}/nbhoods.geojson')
nbhoods.plot()

# ### ID Joins
#
# Join neighbourhood level datasets by neighbourhood ID.

# +
to_join = [
    'crime',
    'education',
    'environment',
    'demographics',
    'housing',
    'economics',
    'civics',
    'health'
]

for name in to_join:
    
    file = Path(f'{dist_folder}/by-dataset/geohoods_to_{name}.geojson')
    if not file.is_file() or force:
        
        # Start processing
        timer = Timer(name, 'join')
        timer.start()

        # Read and prepare data
        data = read_data(f'{dist_folder}/clean-datasets/{name}.geojson')
        data = data.drop('geometry', axis=1)

        # Add neighbourhood and remove from dataset
        if 'neighbourhood' in data:
            nbhoods['neighbourhood'] = data['neighbourhood']
            data = data.drop('neighbourhood', axis=1)

        # Join by nbhood id
        gdf = pd.merge(
            nbhoods,
            data,
            on='neighbourhood_id',
            how='left'
        )
        gdf = gdf.set_index('neighbourhood_id')

        # Save by dataset
        gdf.to_file(file)

        # Save by variable
        save_vars(gdf)

        # End processing
        timer.end()
        
    else:
        print(f'Skipped {name} - already joined ({datetime.now()})')
# -

# Preview results of id joins:

gdf = read_data(f'{dist_folder}/by-dataset/geohoods_to_demographics.geojson')
gdf

# ### Spatial Binning (Counts Only)
#
# Spatially bin counted column values into each neighbourhood using the cleaned datasets.

to_count = [
    'autospeed_enforcement',
    'red_light_cams',
    'police',
    'ambulance',
    'fire_hydrants',
    'fire_stations',
    'renewables',
    'bicycle_parking',
    'transit_shelters',
    'wayfind',
    'litter',
    'art',
    'culture',
    'religious'
]
for name in to_count:
    
    # Read data
    data = read_data(f'{dist_folder}/clean-datasets/{name}.geojson')

    # Only process if dataset file not created
    file = Path(f'{dist_folder}/by-dataset/geohoods_to_{name}.geojson')
    if not file.is_file() or force:

        # Start processing
        timer = Timer(name, 'spatial binning')
        timer.start()

        # Spatial join to grid
        join = nbhoods.sjoin(data, predicate='intersects')

        # Agg counts
        counts = join.groupby(join.index).size()
        counts_col = f'{name}_count'
        counts.name = counts_col

        # Add counts to nbhoods
        gdf = nbhoods.join(counts)
        gdf[counts_col] = gdf[counts_col].fillna(0)

        # Save by dataset
        gdf.to_file(file)

        # Save by variable
        save_vars(gdf)

        # End processing
        timer.end()

    else:
        print(f'Skipping {file} - already binned ({datetime.now()})')

# Preview a spatially binned count dataset:

preview = gpd.read_file(f'{dist_folder}/by-dataset/geohoods_to_autospeed_enforcement.geojson')
preview

preview.plot(
    column='autospeed_enforcement_count',
    edgecolor='gray',
    legend=True
)

# ### Spatial Binning (Aggregates)
#
# Spatially bin aggregate column values into each neighbourhood using the cleaned datasets.

# #### centrelines
#
# Bin centreline geometry calculations and features:

# +
name = 'centrelines'
data = read_data(f'{dist_folder}/clean-datasets/{name}.geojson')

# Only process if dataset file not created
file = Path(f'{dist_folder}/by-dataset/geohoods_to_{name}.geojson')
if not file.is_file() or force:

    # Start processing
    timer = Timer(name, 'spatial binning')
    timer.start()

    # Split lines at polygons using overlay
    overlay = gpd.overlay(
        data,
        nbhoods,
        how='intersection'
    ).explode(index_parts=False)

    # Add sinuosity metric (how curved lines are)
    overlay.insert(
        len(overlay.columns) - 2, # insert before geometry
        'sin',
        lsin(overlay.geometry)
    )

    # Spatial join lines to each nbhood
    join = overlay.sjoin(nbhoods, predicate='intersects')

    # Aggregate cols by nbhood
    group = join.groupby(join['index_right'])
    agg = group.agg({
        'centrelines_feature_code': [mode],
        'centrelines_oneway_dir_code': [mode],
        'geometry': [
            ('len_sum', glen('sum')),
            ('len_mean', glen('mean')),
            ('len_median', glen('median')),
            ('len_min', glen('min')),
            ('len_max', glen('max'))
        ],
        'sin': genstats
    })
    
    # Rename agg cols
    agg.columns = ['_'.join(c).strip().replace('geometry_', '') for c in agg.columns]
    agg.columns = [f'centrelines_{c}' if not c.startswith('centrelines_') else c for c in agg.columns]
    
    # Count freq
    freq = []
    for c in ['centrelines_feature_code', 'centrelines_oneway_dir_code']:
        
        # Get possible unique values in col
        possible = data[c].unique()
        
        # Count freq for each unique value
        f = group.apply(ucounts, c=c, possible=possible)
        
        # Rename freq cols
        f = format_cols(f)
        prefix = c.replace(f'{name}_', '')
        f.columns = [f'centrelines_{prefix}_{c}_count' for c in f.columns]
        freq.append(f)
    
    # Combine count freq
    freq = pd.concat(freq, axis=1)
    freq.index = freq.index.get_level_values(0)

    # Add agg and freq to nbhoods
    gdf = nbhoods.join(agg)
    gdf = gdf.join(freq)

    # Save by dataset
    gdf.to_file(file)

    # Save by variable
    save_vars(gdf)

    # End processing
    timer.end()

else:
    print(f'Skipping {file} - already binned ({datetime.now()})')
# -

# Preview results of spatial binning `centrelines`:

preview = gpd.read_file(f'{dist_folder}/by-dataset/geohoods_to_centrelines.geojson')
preview

preview.plot(
    column='centrelines_feature_code_major_arterial_ramp_count',
    edgecolor='gray',
    legend=True
)

# #### collisions
#
# Bin collision features and counts:

# +
name = 'collisions'
data = read_data(f'{dist_folder}/clean-datasets/{name}.geojson')
    
# Only process if dataset file not created
file = Path(f'{dist_folder}/by-dataset/geohoods_to_{name}.geojson')
if not file.is_file() or force:

    # Start processing
    timer = Timer(name, 'spatial binning')
    timer.start()
    
    # Spatial join to nbhood
    join = nbhoods.sjoin(data, predicate='intersects')

    # Agg cols by nbhood
    group = join.groupby(join.index)
    dtstats = [s for s in genstats if s != 'sum']
    agg = group.agg({
        'collisions_month': [mode],
        'collisions_day_of_month': [mode] + dtstats,
        'collisions_day_of_week': [mode],
        'collisions_year': [mode] + dtstats,
        'collisions_hour': [mode] + dtstats,
        'collisions_fatalities': genstats,
        'collisions_injury': genstats,
        'collisions_ftr': genstats,
        'collisions_pd': genstats,
    })
    
    # Rename agg cols
    agg.columns = ['_'.join(c).strip() for c in agg.columns]
    
    # Count freq
    freq = []
    for c in ['collisions_month', 'collisions_day_of_month', 'collisions_day_of_week', 'collisions_year', 'collisions_hour']:
        
        # Get possible unique values in col
        if c == 'collisions_month':
            possible = ['January', 'February', 'March', 'April', 'May', 'June', 'July', 'August', 'September', 'October', 'November', 'December']
        elif c == 'collisions_day_of_month':
            possible = [*range(1,32)]
        elif c == 'collisions_day_of_week':
            possible = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday']
        elif c == 'collisions_hour':
            possible = [*range(0,24)]
        else:
            possible = data[c].unique()
        
        # Count freq for each unique value
        f = group.apply(ucounts, c=c, possible=possible)
        
        # Rename freq cols
        f = format_cols(f)
        prefix = c.replace(f'{name}_', '')
        f.columns = [f'collisions_{prefix}_{c}_count' for c in f.columns]
        freq.append(f)
    
    # Combine count freq
    freq = pd.concat(freq, axis=1)
    freq.index = freq.index.get_level_values(0)
    
    # Agg counts
    counts = group.size()
    counts.name = 'collisions_count'

    # Add agg, freq, and counts to nbhoods
    gdf = nbhoods.join(agg)
    gdf = gdf.join(freq)
    gdf = gdf.join(counts)
    gdf['collisions_count'] = gdf['collisions_count'].fillna(0)

    # Save by dataset
    gdf.to_file(file)

    # Save by variable
    save_vars(gdf)

    # End processing
    timer.end()

else:
    print(f'Skipping {file} - already binned ({datetime.now()})')
# -

# Preview results of spatial binning `collisions`:

preview = gpd.read_file(f'{dist_folder}/by-dataset/geohoods_to_collisions.geojson')
preview = preview[preview['collisions_fatalities_mean'] > 0]
preview

preview.plot(
    column='collisions_count',
    edgecolor='gray',
    legend=True
)

# #### traffic
#
# Bin counts of traffic features and counts:

# +
name = 'traffic'
data = read_data(f'{dist_folder}/clean-datasets/{name}.geojson')
    
# Only process if dataset file not created
file = Path(f'{dist_folder}/by-dataset/geohoods_to_{name}.geojson')
if not file.is_file() or force:

    # Start processing
    timer = Timer(name, 'spatial binning')
    timer.start()
    
    # Spatial join to nbhood
    join = nbhoods.sjoin(data, predicate='intersects')

    # Agg cols by nbhood
    group = join.groupby(join.index)
    agg = group.agg({
        c: genstats for c in data.columns if c not in ['geometry', 'traffic_month', 'traffic_day_of_month', 'traffic_year']
    })
    
    # Rename agg cols
    agg.columns = ['_'.join(c).strip() for c in agg.columns]
    
    # Count freq
    freq = []
    for c in ['traffic_month', 'traffic_day_of_month', 'traffic_year']:
        
        # Get possible unique values in col
        if c == 'traffic_month':
            possible = [*range(1,32)]
        elif c == 'traffic_day_of_month':
            possible = [*range(1,32)]
        else:
            possible = data[c].unique()
        
        # Count freq for each unique value
        f = group.apply(ucounts, c=c, possible=possible)
        
        # Rename freq cols
        f = format_cols(f)
        prefix = c.replace(f'{name}_', '')
        f.columns = [f'traffic_{prefix}_{c}_count' for c in f.columns]
        freq.append(f)
        
    # Combine count freq
    freq = pd.concat(freq, axis=1)
    freq.index = freq.index.get_level_values(0)
    
    # Agg counts
    counts = group.size()
    counts.name = 'traffic_count'

    # Add agg, freq, and counts to nbhoods
    gdf = nbhoods.join(agg)
    gdf = gdf.join(freq)
    gdf = gdf.join(counts)
    gdf['traffic_count'] = gdf['traffic_count'].fillna(0)

    # Save by dataset
    gdf.to_file(file)

    # Save by variable
    save_vars(gdf)

    # End processing
    timer.end()

else:
    print(f'Skipping {file} - already binned ({datetime.now()})')
# -

# Preview results of spatial binning `traffic`:

preview = gpd.read_file(f'{dist_folder}/by-dataset/geohoods_to_traffic.geojson')
preview = preview[preview['traffic_nb_cars_t_mean'] >= 0]
preview

preview.plot(
    column='traffic_nb_cars_t_mean',
    edgecolor='gray',
    legend=True
)

# #### watch_your_speed
#
# Bin watch your speed program location counts and features:

# +
name = 'watch_your_speed'
data = read_data(f'{dist_folder}/clean-datasets/{name}.geojson')
    
# Only process if dataset file not created
file = Path(f'{dist_folder}/by-dataset/geohoods_to_{name}.geojson')
if not file.is_file() or force:

    # Start processing
    timer = Timer(name, 'spatial binning')
    timer.start()
    
    # Spatial join to nbhood
    join = nbhoods.sjoin(data, predicate='intersects')

    # Agg cols by nbhood
    group = join.groupby(join.index)
    agg = group.agg({
        'wys_dir': [mode],
        'wys_min_speed': [mode] + genstats,
        'wys_speed_limit': [mode] + genstats,
    })
    
    # Rename agg cols
    agg.columns = ['_'.join(c).strip() for c in agg.columns]
    
    # Count freq
    freq = []
    for c in ['wys_dir', 'wys_min_speed', 'wys_speed_limit']:
        
        # Get possible unique values in col
        possible = data[c].unique()
        
        # Count freq for each unique value
        f = group.apply(ucounts, c=c, possible=possible)
        
        # Rename freq cols
        f = format_cols(f)
        prefix = c.replace(f'wys_', '')
        f.columns = [f'wys_{prefix}_{c}_count' for c in f.columns]
        freq.append(f)
        
    # Combine count freq
    freq = pd.concat(freq, axis=1)
    freq.index = freq.index.get_level_values(0)
    
    # Agg counts
    counts = group.size()
    counts.name = 'wys_count'

    # Add agg, freq, and counts to nbhoods
    gdf = nbhoods.join(agg)
    gdf = gdf.join(freq)
    gdf = gdf.join(counts)
    gdf['wys_count'] = gdf['wys_count'].fillna(0)

    # Save by dataset
    gdf.to_file(file)

    # Save by variable
    save_vars(gdf)

    # End processing
    timer.end()

else:
    print(f'Skipping {file} - already binned ({datetime.now()})')
# -

# Preview results of spatial binning `watch_your_speed`:

preview = gpd.read_file(f'{dist_folder}/by-dataset/geohoods_to_watch_your_speed.geojson')
preview = preview[preview['wys_count'] > 0]
preview

preview.plot(
    column='wys_count',
    edgecolor='gray',
    legend=True
)

# #### schools
#
# Bin school location counts and features:

# +
name = 'schools'
data = read_data(f'{dist_folder}/clean-datasets/{name}.geojson')
    
# Only process if dataset file not created
file = Path(f'{dist_folder}/by-dataset/geohoods_to_{name}.geojson')
if not file.is_file() or force:

    # Start processing
    timer = Timer(name, 'spatial binning')
    timer.start()
    
    # Spatial join to nbhood
    join = nbhoods.sjoin(data, predicate='intersects')

    # Agg cols by nbhood
    group = join.groupby(join.index)
    agg = group.agg({
        'schools_level': [mode],
        'schools_type': [mode]
    })
    
    # Rename agg cols
    agg.columns = ['_'.join(c).strip() for c in agg.columns]
    
    # Count freq
    freq = []
    for c in ['schools_level', 'schools_type']:
        
        # Get possible unique values in col
        possible = data[c].unique()
        
        # Count freq for each unique value
        f = group.apply(ucounts, c=c, possible=possible)
        
        # Rename freq cols
        f = format_cols(f)
        prefix = c.replace(f'{name}_', '')
        f.columns = [f'schools_{prefix}_{c}_count' for c in f.columns]
        freq.append(f)
        
    # Combine count freq
    freq = pd.concat(freq, axis=1)
    freq.index = freq.index.get_level_values(0)
    
    # Agg counts
    counts = group.size()
    counts.name = 'schools_count'

    # Add agg, freq, and counts to nbhoods
    gdf = nbhoods.join(agg)
    gdf = gdf.join(freq)
    gdf = gdf.join(counts)
    gdf['schools_count'] = gdf['schools_count'].fillna(0)

    # Save by dataset
    gdf.to_file(file)

    # Save by variable
    save_vars(gdf)

    # End processing
    timer.end()

else:
    print(f'Skipping {file} - already binned ({datetime.now()})')
# -

# Preview results of spatial binning `watch_your_speed`:

preview = gpd.read_file(f'{dist_folder}/by-dataset/geohoods_to_schools.geojson')
preview = preview[preview['schools_count'] > 0]
preview

preview.plot(
    column='schools_count',
    edgecolor='gray',
    legend=True
)

# #### childcare
#
# Bin childcare location counts and features:

# +
name = 'childcare'
data = read_data(f'{dist_folder}/clean-datasets/{name}.geojson')
    
# Only process if dataset file not created
file = Path(f'{dist_folder}/by-dataset/geohoods_to_{name}.geojson')
if not file.is_file() or force:

    # Start processing
    timer = Timer(name, 'spatial binning')
    timer.start()
    
    # Spatial join to nbhood
    join = nbhoods.sjoin(data, predicate='intersects')

    # Agg cols by nbhood
    group = join.groupby(join.index)
    agg = group.agg({
        'childcare_auspice': [mode],
        'childcare_bldg_type': [mode],
        'childcare_igspace': genstats,
        'childcare_tgspace': genstats,
        'childcare_pgspace': genstats,
        'childcare_kgspace': genstats,
        'childcare_sgspace': genstats,
        'childcare_totspace': genstats,
        'childcare_subsidy': genstats
    })
    
    # Rename agg cols
    agg.columns = ['_'.join(c).strip() for c in agg.columns]
    
    # Count freq
    freq = []
    for c in ['childcare_auspice', 'childcare_bldg_type']:
        
        # Get possible unique values in col
        possible = data[c].unique()
        
        # Count freq for each unique value
        f = group.apply(ucounts, c=c, possible=possible)
        
        # Rename freq cols
        f = format_cols(f)
        prefix = c.replace(f'{name}_', '')
        f.columns = [f'childcare_{prefix}_{c}_count' for c in f.columns]
        freq.append(f)
        
    # Combine count freq
    freq = pd.concat(freq, axis=1)
    freq.index = freq.index.get_level_values(0)
    
    # Agg counts
    counts = group.size()
    counts.name = 'childcare_count'

    # Add agg, freq, and counts to nbhoods
    gdf = nbhoods.join(agg)
    gdf = gdf.join(freq)
    gdf = gdf.join(counts)
    gdf['childcare_count'] = gdf['childcare_count'].fillna(0)

    # Save by dataset
    gdf.to_file(file)

    # Save by variable
    save_vars(gdf)

    # End processing
    timer.end()

else:
    print(f'Skipping {file} - already binned ({datetime.now()})')
# -

# Preview results of spatial binning `watch_your_speed`:

preview = gpd.read_file(f'{dist_folder}/by-dataset/geohoods_to_childcare.geojson')
preview = preview[preview['childcare_count'] > 0]
preview

preview.plot(
    column='childcare_count',
    edgecolor='gray',
    legend=True
)

# ### Combine Datasets
#
# Combine processed datasets into one final dataset.

# +
# Get dataset files
dfiles = Path(f'{dist_folder}/by-dataset').glob('*')
dfiles = [f for f in dfiles if f.is_file() and f.suffix == '.geojson']

# Combine if not exists
file = Path(f'{dist_folder}/geohoods_to.geojson')
if not file.is_file() or force:
    
    # Read each file and combine to nbhoods
    out = read_data(f'{dist_folder}/nbhoods.geojson')
    for f in dfiles:
        
        # Start processing
        timer = Timer(str(f), 'combining')
        timer.start()
        
        # Read dataset and drop unneeded columns
        data = read_data(f).drop('geometry', axis=1)
        if 'neighbourhood' in data:
            data = data.drop('neighbourhood', axis=1)
        
        # Combine dataset
        out = pd.merge(
            out,
            data,
            on='neighbourhood_id',
            how='left'
        )
        
        # End processing
        timer.end()
    
    # Save combined datasets
    out.to_file(file)
    
else:
    print(f'Skipping {file} - already combined ({datetime.now()})')
# -

# ## Summary

gtimer.end()

# Information for the final dataset `geohoods_to.geojson`:

out = read_data(f'{dist_folder}/geohoods_to.geojson')
geohoods_info = dict(file=f'{dist_folder}/geohoods_to.geojson')
geohoods_info = dict(geohoods_info, **df_info(out, 'geohoods_to'))
geohoods_info = dict(geohoods_info, **file_info(f'{dist_folder}/geohoods_to.geojson'))
for k, v in geohoods_info.items():
    print(f'{k}: {v}')

# The final dataset is also split by datasets for convenience:

# +
# Get files and dataset names
sources = read_data(data_sources_file)
datasets = [Path(row.file).with_suffix('').stem for row in sources.itertuples() if Path(row.file).stem != study_area_dataset]
files = [f'{dist_folder}/by-dataset/geohoods_to_{name}.geojson' for name in datasets]

# Read datasets
datasets = {name: read_data(f'{dist_folder}/by-dataset/geohoods_to_{name}.geojson') for name in datasets}

# Create summary info
info = pd.DataFrame([df_info(df, name) for name, df in datasets.items()])
info_files = pd.DataFrame([file_info(name) for name in files])
summary = pd.concat([info, info_files], axis=1)

# Add supplymentary info
sources = sources.rename(columns={'file': 'dataset'})
sources = sources[sources['dataset'] != f'{study_area_dataset}.geojson']
sources['dataset'] = [k for k in datasets]
summary = pd.merge(
    summary,
    sources,
    on='dataset',
    how='left'
)
                    
# Rearrange summary columns and display
summary['file'] = files
summary = summary.rename(columns={'url': 'raw_url'})
summary = summary[['dataset', 'file', 'created', 'modified', 'columns', 'rows', 'geometry', 'memory_usage', 'size', 'source', 'source_url', 'raw_url']]
display(HTML(summary.to_html(render_links=True)))

StarcoderdataJupyterScriptsDedupFiltered