import csv
from lxml import html
import time
import requests

"""
Make it work, make it right, make it fast

Extract the tickers from the S&P 500 table on Wikipedia, process them into
a list and save them into a CSV file.

# Retrieve HTML from URL with requests
http://docs.python-requests.org/en/master/user/quickstart/

# HTML table structure
http://www.w3schools.com/html/html_tables.asp

# Python HTML scraping
http://docs.python-guide.org/en/latest/scenarios/scrape/

# HTML table parsing with xpath
http://www.w3schools.com/xml/xpath_syntax.asp

# Save to CSV
http://gis.stackexchange.com/a/72476
"""

url = 'https://en.wikipedia.org/wiki/List_of_S%26P_500_companies'

csv_output = 'sp500_tickers.csv'

start_time = time.time()

# Download the S&P 500 table from Wikipedia, creating a string of the raw HTML
raw_html = requests.get(url).content
html_string = html.fromstring(raw_html)

ticker_list = []

# Pull first HTML table out of the HTML string, then loop through each HTML row
for html_row in html_string.xpath('//table[1]'):
    # Pull each HTML row's code that starts with a <tr> flag
    for col in html_row.xpath('.//tr'):
        # Create a list of text values from each column in this HTML row
        table_row_list = [item.text_content() for item in col.xpath('.//td')]
        # Only process table row lists that have values
        if table_row_list:
            # Tickers are in the first column in the row (first list element)
            ticker = table_row_list[0].strip()
            # Append each row's ticker to the ticker list
            ticker_list.append(ticker)

# Alphabetize ticker list
ticker_list.sort()

print(ticker_list)

# Save the ticker list to a csv file
with open(csv_output, 'w', newline='') as file:
    writer = csv.writer(file)
    for ticker in ticker_list:
        writer.writerow([ticker])

end_time = time.time()
run_time = round(end_time - start_time, 2)
print('Finished extracting the S&P 500 ticker list in %s seconds' % run_time)

Add sp500_extractor_v1 version that does not use BeautifulSoup.
Add sp500_extractor_v1 version that does not use BeautifulSoup.

This version uses the lxml library to parse the HTML string via xpath. Also change urllib to requests for better efficiency. Update initial comment links to show relevant code resources.
