# Import all the things!
import urllib.request
from datetime import *
from lxml import html
from bs4 import BeautifulSoup

"""
Explanation: Scraping examples
Some BeautifulSoup examples to help you scrape the HTML from web pages.
See BeautifulSoup documentation for technical reference.
1. Import libraries
End of explanation
"""


# Scrape all HTML from webpage.
def scrapewebpage(url):
	# Open URL and get HTML.
	web = urllib.request.urlopen(url)

	# Make sure there wasn't any errors opening the URL.
	if (web.getcode() == 200):
		html = web.read()
		return(html)
	else:
		print("Error %s reading %s" % str(web.getcode()), url)

# Helper function that scrape the webpage and turn it into soup.
def makesoup(url):
	html = scrapewebpage(url)
	return(BeautifulSoup(html, "lxml"))

"""
Explanation: 2. Define scraping functions
End of explanation
"""


# Scrape Wikipedia main page.
wp_soup = makesoup("https://en.wikipedia.org/wiki/Main_Page")

# Match the <h2> tag with id the id mp-itn-h2
h2 = wp_soup.find(id="mp-itn-h2")

h2

# Only get the text inside <h2>.
h2.get_text()

"""
Explanation: 3. BeautifulSoup examples
A) Find an id
Use find(id="name") to find the first HTML tag that has an id attribute like this: &lt;h2 id="mp-itn-h2"&gt;&lt;/h2&gt;
End of explanation
"""


# Scrape Wikipedia main page.
wp_soup = makesoup("https://en.wikipedia.org/wiki/Main_Page")

# Find the first HTML tag that has class mw-headline.
headline = wp_soup.find("", "mw-headline")

headline

# Only get the text inside the <span>.
headline.get_text()

"""
Explanation: B) Find a class
Use find("", "name") to find the first HTML tag that has a class attribute like this: &lt;h2 class="name"&gt;&lt;/h2&gt;
End of explanation
"""


# Scrape Wikipedia main page.
wp_soup = makesoup("https://en.wikipedia.org/wiki/Main_Page")

# Find all HTML tag that has class mw-headline.
all_headlines = wp_soup.find_all("", "mw-headline")

all_headlines

# Now we have a list that we can use a for loop.
for headline in all_headlines:
    headline = headline.get_text()
    print(headline)

"""
Explanation: C) Find everything with a class
Use find_all("", "name") to find all HTML tags that has a class attribute like this: &lt;h2 class="name"&gt;&lt;/h2&gt;
End of explanation
"""


# Scrape Wikipedia main page.
wp_soup = makesoup("https://en.wikipedia.org/wiki/Main_Page")

# Find all HTML tag that has class mw-headline.
all_h3 = wp_soup.find_all("h3")

all_h3

# Now we have a list that we can use a for loop.
for h3 in all_h3:
    h3 = h3.get_text()
    print(h3)

"""
Explanation: D) Find all &lt;h3&gt;
Use find_all("h3") to get all &lt;h3&gt; (or something else).
End of explanation
"""


# Scrape a Wikipedia page with a table.
champ_soup = makesoup("https://en.wikipedia.org/wiki/European_Road_Championships")

# Find <table class="wikitable">.
table = champ_soup.find("table", "wikitable")

table

# Go through each row and take the text from 1st and 2nd column.
rows = table.find_all('tr')
for row in rows:
    cols = row.find_all('td')
    if len(cols) > 0:
        Year = cols[0].get_text()        # Get the text in the 1st column.
        Country = cols[1].get_text()     # Get the text in the 2nd column.

        print(Year + " " + Country)

"""
Explanation: E) Find all column values in a table
Use for to loop through a &lt;table&gt; and extract all column values.
End of explanation
"""


table = champ_soup.find("table", "wikitable")

# Get cell value from row 5, column 1.
cell = table.find_all('tr')[5].find_all('td')[1].get_text()

cell

"""
Explanation: F) Find a specific cell value in a table
Use this if you know the row and column of the &lt;table&gt; of the information you want to extract.
End of explanation
"""


# Scrape Wikipedia main page.
wp_soup = makesoup("https://en.wikipedia.org/wiki/Main_Page")

# Find <table class="mp-middle">.
middle_table = wp_soup.find("table", id="mp-middle")

# In the <table>, find <h2>.
h2 = middle_table.find("h2")

h2

# Only get the text inside the <h2>.
h2.get_text()

"""
Explanation: G) Find nested things
Use this if you want to find things that are nested inside each oher.
End of explanation
"""


