# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3 (ipykernel)
#     language: python
#     name: python3
# ---

# # Web scraping example 1: Downloading air quality data from Berlin
#
# Here we use the [Berlin air quality measurement network](https://luftdaten.berlin.de/lqi) as an example. If you manually download data from this website, you can only save data at a single station on a single day at each time. It is time consuming to manually download multiple days of data from each site. In the end, you will have a big number of CSV files in your folder. And there is always a chance of making mistakes.

# ## First let's manually download data at a single station on a single day
# Here is the link to a station called "Wedding background": https://luftdaten.berlin.de/station/mc010
#
# If you open it, you will see the most recent air quality data and the "Save data as CSV" botton. If you left-click this button, you will manually download the data. 
#
# ## Now let's use Python to download the same data
#
# If you right-click this "Save data as CSV" botton, then select "Inspect", a lot of codes will appear at the bottom of your browser. You will see a highlighted url containing information of the station code, starting date and end date. Then right-click on this url and select "Copy link address", you will get a complete url like this one:
#
# ```
# https://luftdaten.berlin.de/station/mc010.csv?group=pollution&period=1h&timespan=custom&start%5Bdate%5D=26.12.2021&start%5Bhour%5D=22&end%5Bdate%5D=27.12.2021&end%5Bhour%5D=22
# ```
#
# I did this on 27/12/2021, so the input dates in the above url were the most recent dates (26-27 December 2021) by default.

# +
# load packages
import pandas as pd
from urllib.request import urlopen

# # copy and paste the url from above
target_url = "https://luftdaten.berlin.de/station/mc010.csv?group=pollution&period=1h&timespan=custom&start%5Bdate%5D=26.12.2021&start%5Bhour%5D=22&end%5Bdate%5D=27.12.2021&end%5Bhour%5D=22"

# open the target url
url_response = urlopen(target_url)

# read data from the response (the seperator may not always be ";" for other websites)
data_downloaded = pd.read_csv(url_response,sep=";")

# check the results
display(data_downloaded)
# -

# ## Learning from the above example, we can edit inputs and download data at a specified station during a custom time period

# +
# provide a station code
station_code = "171"

# provide the starting date and hour
start_y = 2021
start_m = 11
start_d = 21
start_h = 15

# provide the ending date and hour
end_y = 2021
end_m = 12
end_d = 24
end_h = 23

# construct the target url with inputs from above
# in order to display the whole url on GitHub, I used multiple lines here
# you can remove "\" at end of each line and just use a single line
target_url = "https://luftdaten.berlin.de/station/mc"+str(station_code)+ \
             ".csv?group=pollution&period=1h&timespan=custom&start%5Bdate%5D="+ \
             str(start_d)+"."+str(start_m)+"."+str(start_y)+"&start%5Bhour%5D="+str(start_h)+ \
             "&end%5Bdate%5D="+str(end_d)+"."+str(end_m)+"."+str(end_y)+"&end%5Bhour%5D="+str(end_h)

# open the target url
url_response = urlopen(target_url)

# read data from the response (the seperator may not always be ";" for other websites)
data_downloaded = pd.read_csv(url_response,sep=";")

# check the results
display(data_downloaded)


# -

# ## Wrap all these steps in a function

def download_Berlin_data(station_code,start,end):
    '''Input the station code from Berlin air quality measurement network, the start and end of the study period,
       return data at the target site during the specified time period in a pandas data frame.
    '''
    from urllib.request import urlopen
    import pandas as pd
    # prepare the input information for use in the url
    station_code = str(station_code)
    start_datetime = pd.to_datetime(start)
    end_datetime = pd.to_datetime(end)
    # construct the target url with inputs from above
    # to display the whole url on GitHub I used multiple lines, you can just use a single line
    target_url = "https://luftdaten.berlin.de/station/mc"+str(station_code)+ \
                 ".csv?group=pollution&period=1h&timespan=custom&start%5Bdate%5D="+ \
                 str(start_datetime.day)+"."+str(start_datetime.month)+"."+str(start_datetime.year)+ \
                 "&start%5Bhour%5D="+str(start_datetime.hour)+ "&end%5Bdate%5D="+str(end_datetime.day)+"."+ \
                 str(end_datetime.month)+"."+str(end_datetime.year)+"&end%5Bhour%5D="+str(end_datetime.hour)
    # open the target url
    url_response = urlopen(target_url)
    # read data from the response (the seperator may not always be ";" for other websites)
    data_downloaded = pd.read_csv(url_response,sep=";")
    return data_downloaded


# test our function
download_Berlin_data("010","2021-10-10 02:00","2021-10-11 15:00")

# ## Download data from multiple sites and days all at once

# +
# get data at these three sites during the same time period
station_codes = ["010","027","117"]

Berlin_data = []

for station in station_codes:
    Berlin_data.append(download_Berlin_data(station,"2021-9-10 00:00","2021-10-10 23:00"))
# -

# check results at the "010" site
display(Berlin_data[0])

# check results at the "027" site
display(Berlin_data[1])

# check results at the "117" site
display(Berlin_data[2])

# ## These codes work for the [Berlin air quality measurement network](https://luftdaten.berlin.de/lqi), but they may not work for other websites. You can get an idea of web scraping here and develop your codes when needed. The [urllib](https://docs.python.org/3/library/urllib.html) is not the only Python package for web scraping. In some complicated cases, you may want to try [Selenium](https://selenium-python.readthedocs.io/) or [Scrapy](https://scrapy.org/). 

StarcoderdataJupyterScriptsDedupFiltered