import requests
from lxml import html

"""
Explanation: Import needed libraries
End of explanation
"""


response = requests.get('http://news.ycombinator.com/')
response

response.content

"""
Explanation: We used the library "request" last time in getting Twitter data (REST-ful).  We are introducing the new "lxml" library for analyzing & extracting HTML elements and attributes here.
Use Requests to get HackerNews content
HackerNews is a community contributed news website with an emphasis on technology related content.  Let's grab the set of articles that are at the top of the HN list.
End of explanation
"""


page = html.fromstring(response.content)
page

"""
Explanation: We will now use lxml to create a programmatic access to the content from HackerNews.
Analyzing HTML Content
End of explanation
"""


posts = page.cssselect('.title')

len(posts)

"""
Explanation: CSS Selectors
For those of you who are web designers, you are likely very familiar with Cascading Stylesheets (CSS).  Here is an example for how to use CSS selector for finding specific HTML elements
End of explanation
"""


posts = page.xpath('//td[contains(@class, "title")]')

len(posts)

"""
Explanation: Details of how to use CSS selectors can be found in the w3 schools site:
http://www.w3schools.com/cssref/css_selectors.asp
XPath
Alternatively, we can use a standard called "XPath" to find specific content in the HTML.
End of explanation
"""


posts = page.xpath('//td[contains(@class, "title")]/a')

len(posts)

"""
Explanation: We are only interested in those "td" tags that contain an anchor link to the referred article.
End of explanation
"""


first_post = posts[0]
first_post.text

"""
Explanation: So, only half of those "td" tags with "title" contain posts that we are interested in.  Let's take a look at the first such post.
End of explanation
"""


first_post.attrib

first_post.attrib["href"]

all_links = []
for p in posts:
    all_links.append((p.text, p.attrib["href"]))

all_links

"""
Explanation: There is a lot of "content" in the td tag's attributes.
End of explanation
"""


