from bs4 import BeautifulSoup
import urllib
url= 'http://atlanta.backpage.com/'
page = urllib.request.urlopen(url)
soup = BeautifulSoup(page.read(), "lxml")
soup.find('body')

from IPython.display import HTML, display
display(HTML(str(soup.find('body'))))

"""
Explanation: Web Crawling
<img src=https://1.bp.blogspot.com/-i5pB5HhkDSo/V1CMvnT86_I/AAAAAAAAABU/Oth7ddHBJHIaou-jlXV_FJP_i9ZWvvYGQCLcB/s400/WebCrawler.jpg>
One Approach
End of explanation
"""


class TorEngine(DefaultEngine):
    def __init__(self, pw = None, control = ("127.0.0.1", 9051), signal = Signal.NEWNYM,
            proxy_handler = urllib.request.ProxyHandler({"http": "127.0.0.1:8118"}),
            data = None, headers = { "User-Agent": DEFAULT_USER_AGENT }):
        if pw:
            self.pw = pw
        else:
            self.pw = getpass.getpass("Tor password: ")
        self.control = control
        self.signal = signal
        self.proxy_handler = proxy_handler
        proxy_opener = urllib.request.build_opener(self.proxy_handler)
        urllib.request.install_opener(proxy_opener)
        super(TorEngine, self).__init__(data, headers)


"""
Explanation: Problem

Sites might limit the revisit rate from a given IP address

Solution

Change IP addresses

Tor Scaper
<img src='../images/tor.png'>
End of explanation
"""


from palantiri.core import engine
te = engine.TorEngine()

out = te.get_page_source('http://ifconfig.me/ip')
print(out.source)

out = te.get_page_source(url)
print(out.source)

"""
Explanation: ...
<a href=https://github.com/anidata/palantiri/blob/901c72e1541fcce8aef57a0b8c0e7435cf007a86/src/core/engine.py#L54>On GitHub</a>
Install Tor and Privoxy
brew install privoxy
brew install tor
<a href=https://github.com/anidata/palantiri/blob/master/docs/RunWithTor.md>Modify some config files</a>
Start Services
$ brew services start privoxy
==&gt; Successfully started `privoxy` (label: homebrew.mxcl.privoxy)
$ brew services start tor
==&gt; Successfully started `tor` (label: homebrew.mxcl.tor)
End of explanation
"""


import pandas as pd
dfs = pd.read_html('http://www.espn.com/nba/boxscore?gameId=400899810')
dfs[1]

"""
Explanation: Full Source on GitHub
<a href=https://github.com/anidata/palantiri>anidata/palantiri</a>
Bonus
Scraping Tables with Pandas
<a href=http://pandas.pydata.org/pandas-docs/stable/generated/pandas.read_html.html>pandas.read_html</a>
End of explanation
"""


