import pandas as pd
%pylab inline

import requests
from bs4 import BeautifulSoup as bs4

"""
Explanation: <b>------------------------</b><br>
(C) 08/2017 by Andy Mai<br>
Email: andymai.ca@aol.com<br>
San Jose, CA - Capital of Silicon Valley<br>
<b>------------------------</b>
Craigslist - Apartments' Rental Prices in SF Bay Area
Get information from Craigslist to Python via HTML Get and BeautifulShoup API
In this example, we will get information about apartments' rental prices in SF Bay Area (California)
End of explanation
"""


url_base = 'https://sfbay.craigslist.org/search/apa'
params = dict(bedrooms=1, is_furnished=1)
resq = requests.get(url_base, params=params)
# We have the full URL:
print (resq.url)

"""
Explanation: We will do step by step how to get information from Craigslist.
Assume that we would like to get data of apartments with 01 bedroom and already furnished
STEP 01: Prepare the URL in order to GET information

The base will be stored in <strong>url_base</strong> variable
The extension for HTML GET will be stored in <b>params</b>
<br>After that you can see the full URL will be passed as below:
<br><i>https://sfbay.craigslist.org/search/apa?s=120&is_furnished=1&bedrooms=1
</i>
End of explanation
"""


html = bs4(resq.text, 'html.parser')
print(html.prettify()[:900])

"""
Explanation: We will use BeautifulSoup API to simplify the content of what we get from Craigslist
End of explanation
"""


apts = html.find_all('p', attrs={'class':'result-info'})
print(len(apts))

"""
Explanation: STEP 02: Get necessary information from the web
We can see that every single post will be stored in <b>p</b> attribute with class <b>result-info</b><br>
From that observation, we can retrieve all information by getting them from this attribute.
End of explanation
"""


this_aprt = apts[15]
print(this_aprt)

"""
Explanation: <b>apts</b> is a list of 120 renting objects. We can take one out for our sample. We work on this sample to have the sense how to apply all these things to the whole list.
End of explanation
"""


size = this_aprt.find_all(attrs={'class':'housing'})[0].text
print(size)

"""
Explanation: We get the information about number of bedroom and area
End of explanation
"""


def size_brs(size):
    split_var = size.strip('\n ').strip('\n - ').split('-')
    if len(split_var) == 2:
        n_brs = split_var[0].replace('br', '')
        apt_size = split_var[1].replace('ft2', '')
    elif 'br' in split_var[0]:
        n_brs = split_var[0].replace('br', '')
        apt_size = np.nan
    elif 'ft2' in split_var[1]:
        apt_size = split_var[1].replace('ft2', '')
        n_brs = np.nan
    return float(apt_size), float(n_brs)

apt_size, n_brs = size_brs(size)

time = this_aprt.find('time')['datetime']
time = pd.to_datetime(time)

price = float(this_aprt.find('span', {'class':'result-price'}).text.strip('$'))

title = this_aprt.find('a', {'class':'result-title hdrlnk'}).text

link = this_aprt.find('a', {'class':'result-title hdrlnk'})['href']

"""
Explanation: We will create a function to get those information and store into apt_size and n_brs variables:
End of explanation
"""


print('\n'.join([str(i) for i in [apt_size, n_brs, time, price, title]]))

"""
Explanation: STEP 03: Put everything into an array then we have done!
We have all relevant information we need to know about apartments' rental price in SF Bay Area. We just display it to make sure that everything is correct.
End of explanation
"""


page_index = np.arange(0, 1000, 120)

def set_price(x): # Avoid getting None value
    price = x.find('span', {'class':'result-price'})
    if price is not None:
        price = float(price.text.strip('$'))
    else:
        price = np.nan
    return price

def set_time(x): # Avoid getting None value
    time = x.find('time')['datetime']
    if time is not None:
        time = pd.to_datetime(time)
    else:
        time = np.nan
    return time

"""
Explanation: Apply everything will did above to the whole list of rental objects.

Object: is every single apartment posted on Craigslist that we are working on.
Each page will show 120 objects by default.
We can iterate over 1000 objects to get the information
End of explanation
"""


import time as tp
result = []
for j in page_index:
    url_base = 'https://sfbay.craigslist.org/search/apa'
    params = dict(bedrooms=1, is_furnished=1,s=j)
    resq = requests.get(url_base, params=params)
    html = bs4(resq.text, 'html.parser')
    apts = html.find_all('p', attrs={'class':'result-info'})
    for this_aprt in apts:
        size = this_aprt.find_all(attrs={'class':'housing'})[0].text
        apt_size, n_brs = size_brs(size)
        time = set_time(this_aprt)
        price = set_price(this_aprt)
        title = this_aprt.find('a', {'class':'result-title hdrlnk'}).text
        link = this_aprt.find('a', {'class':'result-title hdrlnk'})['href']
        data = np.array([time, price, apt_size, n_brs,title, link])
        result.append(data)
    tp.sleep(1)

"""
Explanation: Some notices in this step:

We need to have a delay when we query Craigslist because if we keep querying too often, our IP address will be blocked (to avoid DDoS attack). In this step, I set the delay time is 1 second for each query.
In params, <b>s</b> parameter indicates the page we are currently working on. By increasing this variable, we will iterate over 1000 objects via multiple pages.
After everything's done, we store the result into <b>result</b> array. This is the input data for us to put in pandas DataFrame for later analytics purpose.
End of explanation
"""


col_names = ['time', 'price', 'size', 'brs', 'title', 'link']
result = pd.DataFrame(data=result, columns=col_names)
result.set_index('time')
result.head()

import matplotlib.pyplot as plt
import seaborn as sns
%matplotlib inline

plt.figure(figsize=(12,10))
sns.distplot(result.price.dropna(),bins=np.arange(0, 10000, 100),color='red',axlabel='SF Bay Area - Apartments\' Rental Price ($)').set(xlim=(0, 12000))

"""
Explanation: Put everything into Pandas DataFrame
End of explanation
"""


