# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import time
import os
import re
import geopandas as gpd
from functions import getTradingPartners
from functions import getData

basePath = 'C:/Users/Krist/University College London/Digital Visualisation/Final Project/'
countries = gpd.read_file(basePath+'Data Sources/countries (1)/ne_50m_admin_0_countries.shp').fillna(value='None')
airports = pd.read_csv(basePath+'Final Data/airports.csv',index_col=0)
ports = pd.read_csv(basePath+'Final Data/ports.csv',index_col=0)
globalData = pd.read_csv(basePath+'Final Data/globalData_reCat.csv',index_col=0)
ukports = pd.read_csv(basePath+'Final Data/UKports_kln.csv',index_col=0)
ukairports = pd.read_csv(basePath+'Final Data/UKFreightAirports.csv',index_col=0)

ukairports


def cleanName(whatToClean,remove=None):
    
    if type(whatToClean) == list:
        
        cleaned = []
        
        for element in whatToClean:
            if remove != None:
                cleaned.append(" ".join([ele.capitalize() for ele in re.split(' ',element) if ele != remove]))
            else:
                cleaned.append(" ".join([ele.capitalize() for ele in re.split(' ',element)]))
        print(cleaned)
        
    else:
        if remove != None:
            cleaned = " ".join([ele.capitalize() for ele in re.split(' ',whatToClean) if ele != remove])
        else:
            cleaned = " ".join([ele.capitalize() for ele in re.split(' ',whatToClean)])
        
    return cleaned 


# ### Getting a sense of that the datasets contains

airports.columns

airports.usage.unique()

airports['size'].unique()

ports.columns

ports.harborsize.unique()

ports.fillna('None').railway.unique()

ports.railway.unique()

countries.columns

globalData.columns

globalData.loc[0]

print('Maximum:',globalData.all_commodities_export.max(),'\nMinimum:',globalData.all_commodities_export.min())

plt.hist(globalData.all_commodities_export)
plt.show()

plt.hist(np.log(globalData.all_commodities_export))
plt.show()

print('Maximum:',globalData.normalised_export_2017.max(),'\nMinimum:',globalData.normalised_export_2017.min())

print('Maximum:',globalData.all_commodities_import.max(),'\nMinimum:',globalData.all_commodities_import.min())

plt.hist(globalData.all_commodities_import)
plt.show()

print('Maximum:',globalData.normalised_import_2017.max(),'\nMinimum:',globalData.normalised_import_2017.min())

plt.hist(np.log(1+globalData.all_commodities_import))
plt.show()

print('Maximum:',globalData.passengers_2017.max(),'\nMinimum:',globalData.passengers_2017.min())

plt.hist(globalData.passengers_2017)
plt.show()

plt.hist(np.log(1+globalData.passengers_2017))
plt.show()

print('Maximum:',globalData.normalised_passengers_2017.max(),'\nMinimum:',globalData.normalised_passengers_2017.min())

print('Maximum:',globalData.freight_2017.max(),'\nMinimum:',globalData.freight_2017.min())

plt.hist(globalData.freight_2017)
plt.show()

plt.hist(np.log(1+globalData.freight_2017))
plt.show()

print('Maximum:',globalData.normalised_freight_2017.max(),'\nMinimum:',globalData.normalised_freight_2017.min())

# ### Let's write some csv's!
#
# #### First the top importers/exporters

globalData

top_five_importers = globalData.sort_values(by=['all_commodities_import'],ascending=False)[['all_commodities_import','name','iso3']]
top_five_importers = top_five_importers.reset_index(drop=True).loc[0:4]
top_five_importers = top_five_importers[['name','all_commodities_import','iso3']]
top_five_importers['all_commodities_import'] = top_five_importers['all_commodities_import'].astype(int)
top_five_importers.columns = ['name','value','code']
top_five_importers['name'] = cleanName(list(top_five_importers.name))

top_five_importers

top_five_exporters = globalData.sort_values(by=['all_commodities_export'],ascending=False)[['all_commodities_export','name','iso3']]
top_five_exporters = top_five_exporters.reset_index(drop=True).loc[0:4]
top_five_exporters = top_five_exporters[['name','all_commodities_export','iso3']]
top_five_exporters['all_commodities_export'] = (top_five_exporters['all_commodities_export']).astype(int)
top_five_exporters.columns = ['name','value','code']
top_five_exporters['name'] = cleanName(list(top_five_exporters.name))

top_five_exporters

top_five_importers.to_csv(basePath+'Layers/StylingDataDriven/top_five_importers.csv',header=True,index=False)
top_five_exporters.to_csv(basePath+'Layers/StylingDataDriven/top_five_exporters.csv',header=True,index=False)

# #### Now the top 5 busiest airports

airports.columns

airports.head()

top_five_airports = airports[airports.busiest_airport_ranking!=0].sort_values(by=['busiest_airport_ranking'],ascending=True)\
                    [['airport_name','iata_code','amount_passed_through','busiest_airport_ranking']]
top_five_airports = top_five_airports.reset_index(drop=True).loc[0:4]
top_five_airports = top_five_airports[['airport_name','amount_passed_through','iata_code']]
top_five_airports.columns = ['name','value','code']
top_five_airports['value'] = top_five_airports['value'].round(2)
top_five_airports['name'] = cleanName(list(top_five_airports.name),"Int'l")

top_five_airports

top_five_airports.to_csv(basePath+'Layers/StylingDataDriven/top_five_airports.csv',header=True,index=False)

# #### On to the busiest ports

ports.columns

top_five_ports = ports[ports.busiest_ports_ranking!=0].sort_values(by=['busiest_ports_ranking'],ascending=True)\
                    [['port_name','amount_shipped_through','busiest_ports_ranking']]
top_five_ports = top_five_ports.reset_index(drop=True).loc[0:4]
top_five_ports = top_five_ports[['port_name','amount_shipped_through','port_name']]
top_five_ports.columns = ['name','value','code']
top_five_ports['name'] = cleanName(list(top_five_ports.name))

top_five_ports

top_five_ports.to_csv(basePath+'Layers/StylingDataDriven/top_five_ports.csv',header=True,index=False)

# #### UK airports

ukairports.head()

ukairports.columns

top_five_ukairports = ukairports.sort_values(by=['total_freight'],ascending=False)[['airport_name','total_freight','iata_code']]
top_five_ukairports = top_five_ukairports.reset_index(drop=True).loc[0:4]
top_five_ukairports.columns = ['name','value','code']
top_five_ukairports['value'] = top_five_ukairports['value'].round(2)
top_five_ukairports['name'] = cleanName(list(top_five_ukairports.name),"Int'l")

top_five_ukairports

top_five_ukairports.to_csv(basePath+'Layers/StylingDataDriven/top_five_ukairports.csv',header=True,index=False)

# #### Finally, the UK ports

ukports.head()

ukports.columns

top_five_ukports = ukports.sort_values(by=['Total'],ascending=False)[['Total','port_name','port_name']]
top_five_ukports.index=np.arange(top_five_ukports.shape[0])
top_five_ukports = top_five_ukports.loc[0:4]
top_five_ukports.columns = ['Total','port_name_1','port_name_2']
top_five_ukports = top_five_ukports[['port_name_1','Total','port_name_2']]
top_five_ukports.columns = ['name','value','code']
top_five_ukports['value'] = top_five_ukports['value'].round(2)
top_five_ukports['name'] = cleanName(list(top_five_ukports.name))

top_five_ukports

top_five_ukports.to_csv(basePath+'Layers/StylingDataDriven/top_five_ukports.csv',header=True,index=False)

# Getthe the top-five trading partners of UK
exportPartners, theirShare = getTradingPartners('United Kingdom','2015','export',basePath=basePath)

exportPartners

theirShare

topfiveTradingPartnersExport = pd.DataFrame(index=np.arange(5),columns=['name','value','code'])
topfiveTradingPartnersExport['value'] = [partner[1] for partner in exportPartners]
topfiveTradingPartnersExport['value'] = topfiveTradingPartnersExport['value'].round(2)
topfiveTradingPartnersExport['name'] = [partner[0] for partner in exportPartners]
topfiveTradingPartnersExport['code'] = [[iso3 for iso3,name in zip(countries.GU_A3,countries.NAME)\
                                             if partner[0] in name.lower()][0] for partner in exportPartners]
topfiveTradingPartnersExport['name'] = cleanName(list(topfiveTradingPartnersExport.name))

topfiveTradingPartnersExport

topfiveTradingPartnersExport.to_csv('../Layers/StylingDataDriven/top_five_exporters_uk.csv',header=True,index=True)

# Getthe the top-five trading partners of UK
importPartners, theirShare_I = getTradingPartners('United Kingdom','2015','import',basePath=basePath)

importPartners

theirShare_I

topfiveTradingPartnersImport = pd.DataFrame(index=np.arange(5),columns=['name','value','code'])
topfiveTradingPartnersImport['value'] = [partner[1] for partner in importPartners]
topfiveTradingPartnersImport['value'] = topfiveTradingPartnersImport['value'].round(2)
topfiveTradingPartnersImport['name'] = [partner[0] for partner in importPartners]
topfiveTradingPartnersImport['code'] = [[iso3 for iso3,name in zip(countries.GU_A3,countries.NAME)\
                                             if partner[0] in name.lower()][0] for partner in importPartners]
topfiveTradingPartnersImport['name'] = cleanName(list(topfiveTradingPartnersImport.name))

topfiveTradingPartnersImport

topfiveTradingPartnersImport.to_csv('../Layers/StylingDataDriven/top_five_importers_uk.csv',header=True,index=True)



StarcoderdataJupyterScriptsDedupFiltered