# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: cities
#     language: python
#     name: cities
# ---

# +
import os

import numpy as np
import pandas
import matplotlib.pyplot as plt
# -

major_cities = pandas.read_csv("./raw/most-populous-cities.csv").loc[:,"Name":"Population"]
major_cities

# ### Classification
# Based off of: https://en.wikipedia.org/wiki/Settlement_hierarchy
#
# But the ratios of subsequent categories should be somewhat consistent so we respect the power-law distribution of sizes.
#
# - **Megacities** ("megalopolis") - More than 8m (The convention is 10m, but I wanted slightly more balanced categories)
# - **Large city** ("conurbation") - More than 3m 
# - **Medium city** ("metropolis") - More than 1m
# - **Small city** ("regiopolis") - More than 300k
# - **Tiny city** - More than 100k
# - **Town** - More than 10k
#
# In this first dataset, we only go down to small cities (at just above the 500k mark).

MEGA_CITY_THRESHOLD = 8e6 # extra _ for naming consistency
LARGE_CITY_THRESHOLD = 3e6
MEDIUM_CITY_THRESHOLD = 1e6
SMALL_CITY_THRESHOLD = 3e5
TINY_CITY_THRESHOLD = 1e5
TOWN_THRESHOLD = 1e4


def get_cities_within_range(cities, min_=0, max_=np.inf):
    """
    :param cities: a numpy dataframe of city populations in decreasing order. 
        Populations in a column titled "Population"
        
    """
    cities_pops = cities.loc[:, "Population"].to_numpy()
    
    min_index = len(cities_pops)
    max_index = -1
    
    cities_too_small = np.nonzero(cities_pops < min_)[0]
    cities_too_large = np.nonzero(cities_pops > max_)[0]
        
    if len(cities_too_small):
        min_index = cities_too_small[0]

    if len(cities_too_large):
        max_index = cities_too_large[-1] 
        
    return cities.loc[max_index+1:min_index-1, :]
            


mega_cities = get_cities_within_range(major_cities, min_=MEGA_CITY_THRESHOLD)
large_cities = get_cities_within_range(major_cities, max_=MEGA_CITY_THRESHOLD, min_=LARGE_CITY_THRESHOLD)
medium_cities = get_cities_within_range(major_cities, max_=LARGE_CITY_THRESHOLD, min_=MEDIUM_CITY_THRESHOLD)
small_cities = get_cities_within_range(major_cities, max_=MEDIUM_CITY_THRESHOLD, min_=SMALL_CITY_THRESHOLD)

print("MEGACITIES\n", mega_cities, "\n")
print("LARGE_CITIES\n", large_cities, "\n")
print("MEDIUM_CITIES\n", medium_cities, "\n")
print("SMALL_CITIES\n", small_cities, "\n")

mega_cities.to_csv("./cleaned/mega.csv")
large_cities.to_csv("./cleaned/large.csv")
medium_cities.to_csv("./cleaned/medium.csv")
small_cities.to_csv("./cleaned/small.csv")

StarcoderdataJupyterScriptsDedupFiltered