# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

# # Getting Data

# Browse [data.gov](https://www.data.gov/):
#
# - [climate](https://www.data.gov/climate/)
# - [Data](https://catalog.data.gov/dataset?groups=climate5434&#topic=climate_navigation)
# - Third [HTML](https://www.ncdc.noaa.gov/stormevents/)

# # (Some) More Sources
#
# - [Open Data Census](http://us-cities.survey.okfn.org/)
# - [Humanitarian Data Exchange](https://data.humdata.org/)
# - [curated data set lists](https://github.com/awesomedata/awesome-public-datasets)
# - [curated API lists](https://github.com/abhishekbanthia/Public-APIs)
# - [Ohio Checkbook](http://ohiotreasurer.gov/Transparency/Ohios-Online-Checkbook)
#

# # From Python
#
# Install and import [requests]()

import requests
tb_data_dict_url = 'https://extranet.who.int/tme/generateCSV.asp?ds=dictionary'
resp = requests.get(tb_data_dict_url)

print(resp.text[:500])

import pandas as pd
import io

# +
# pd.read_csv?
# -

buffer = io.StringIO(resp.text)
tb = pd.read_csv(buffer)

tb.head()

# Options available for authentication

# [In case of zipfiles...](https://techoverflow.net/2018/01/16/downloading-reading-a-zip-file-in-memory-using-python/)

import zipfile
def download_extract_zip(url):
    """
    Download a ZIP file and extract its contents in memory
    yields (filename, file-like object) pairs
    """
    response = requests.get(url)
    with zipfile.ZipFile(io.BytesIO(response.content)) as thezip:
        for zipinfo in thezip.infolist():
            with thezip.open(zipinfo) as thefile:
                yield zipinfo.filename, thefile


url='http://datasets.wri.org/dataset/540dcf46-f287-47ac-985d-269b04bea4c6/resource/27c271ef-63c3-49c5-a06a-f21bb7b96371/download/globalpowerplantdatabasev110'
(filename, file_buffer) = next(download_extract_zip(url))

url='http://datasets.wri.org/dataset/540dcf46-f287-47ac-985d-269b04bea4c6/resource/27c271ef-63c3-49c5-a06a-f21bb7b96371/download/globalpowerplantdatabasev110'
for (filename, file_buffer) in download_extract_zip(url):
    print(filename)

url='http://datasets.wri.org/dataset/540dcf46-f287-47ac-985d-269b04bea4c6/resource/27c271ef-63c3-49c5-a06a-f21bb7b96371/download/globalpowerplantdatabasev110'
for (filename, file_buffer) in download_extract_zip(url):
    if filename.endswith('.csv'):
        powerplants = pd.read_csv(file_buffer)

powerplants.head()

# See also: [Selenium](https://realpython.com/modern-web-automation-with-python-and-selenium/)

StarcoderdataJupyterScriptsDedupFiltered