import urllib2
from bs4 import BeautifulSoup

"""
Explanation: 数据抓取：

使用Python编写网络爬虫


王成军
wangchengjun@nju.edu.cn
计算传播网 http://computational-communication.com
需要解决的问题

页面解析
获取Javascript隐藏源数据
自动翻页
自动登录
连接API接口
End of explanation
"""


url = 'file:///Users/chengjun/GitHub/cjc2016/data/test.html'
content = urllib2.urlopen(url).read() 
soup = BeautifulSoup(content, 'html.parser') 
soup

print(soup.prettify())

"""
Explanation: 一般的数据抓取，使用urllib2和beautifulsoup配合就可以了。
尤其是对于翻页时url出现规则变化的网页，只需要处理规则化的url就可以了。
以简单的例子是抓取天涯论坛上关于某一个关键词的帖子。
在天涯论坛，关于雾霾的帖子的第一页是：
http://bbs.tianya.cn/list.jsp?item=free&nextid=0&order=8&k=雾霾
第二页是：
http://bbs.tianya.cn/list.jsp?item=free&nextid=1&order=8&k=雾霾



Beautiful Soup

Beautiful Soup is a Python library designed for quick turnaround projects like screen-scraping. Three features make it powerful:


Beautiful Soup provides a few simple methods. It doesn't take much code to write an application
Beautiful Soup automatically converts incoming documents to Unicode and outgoing documents to UTF-8. Then you just have to specify the original encoding.
Beautiful Soup sits on top of popular Python parsers like lxml and html5lib.

Install beautifulsoup4
open your terminal/cmd

$ pip install beautifulsoup4

第一个爬虫
Beautifulsoup Quick Start 
http://www.crummy.com/software/BeautifulSoup/bs4/doc/
End of explanation
"""


for tag in soup.find_all(True):
    print(tag.name)

soup('head') # or soup.head

soup('body') # or soup.body

soup('title')  # or  soup.title

soup('p')

soup.p

soup.title.name

soup.title.string

soup.title.text

soup.title.parent.name

soup.p

soup.p['class']

soup.find_all('p', {'class', 'title'})

soup.find_all('p', class_= 'title')

soup.find_all('p', {'class', 'story'})

soup.find_all('p', {'class', 'story'})[0].find_all('a')

soup.a

soup('a')

soup.find(id="link3")

soup.find_all('a')

soup.find_all('a', {'class', 'sister'}) # compare with soup.find_all('a')

soup.find_all('a', {'class', 'sister'})[0]

soup.find_all('a', {'class', 'sister'})[0].text

soup.find_all('a', {'class', 'sister'})[0]['href']

soup.find_all('a', {'class', 'sister'})[0]['id']

soup.find_all(["a", "b"])

print(soup.get_text())

"""
Explanation: html
head
title


body
p (class = 'title', 'story' )
a (class = 'sister')
href/id
End of explanation
"""


url = "http://mp.weixin.qq.com/s?__biz=MzA3MjQ5MTE3OA==&\
mid=206241627&idx=1&sn=471e59c6cf7c8dae452245dbea22c8f3&3rd=MzA3MDU4NTYzMw==&scene=6#rd"
content = urllib2.urlopen(url).read() #获取网页的html文本
soup = BeautifulSoup(content, 'html.parser') 
print soup.title.text
print soup.find('div', {'class', 'rich_media_meta_list'}).find(id = 'post-date').text
print soup.find('div', {'class', 'rich_media_content'}).get_text()


"""
Explanation: 数据抓取：

根据URL抓取微信公众号文章内容



王成军
wangchengjun@nju.edu.cn
计算传播网 http://computational-communication.com
End of explanation
"""


from IPython.display import display_html, HTML
HTML('<iframe src=http://bbs.tianya.cn/list.jsp?item=free&nextid=%d&order=8&k=PX width=1000 height=500></iframe>')
# the webpage we would like to crawl

page_num = 0
url = "http://bbs.tianya.cn/list.jsp?item=free&nextid=%d&order=8&k=PX" % page_num
content = urllib2.urlopen(url).read() #获取网页的html文本
soup = BeautifulSoup(content, "lxml") 
articles = soup.find_all('tr')

print articles[0]

print articles[1]

len(articles[1:])

"""
Explanation: 数据抓取：

抓取天涯回帖网络


王成军
wangchengjun@nju.edu.cn
计算传播网 http://computational-communication.com
End of explanation
"""


for t in articles[1].find_all('td'): print t

td = articles[1].find_all('td')

print td[0]

print td[0]

print td[0].text

print td[0].text.strip()

print td[0].a['href']

print td[1]

print td[2]

print td[3]

print td[4]

records = []
for i in articles[1:]:
    td = i.find_all('td')
    title = td[0].text.strip()
    title_url = td[0].a['href']
    author = td[1].text
    author_url = td[1].a['href']
    views = td[2].text
    replies = td[3].text
    date = td[4]['title']
    record = title + '\t' + title_url+ '\t' + author + '\t'+ author_url + '\t' + views+ '\t'  + replies+ '\t'+ date
    records.append(record)

print records[2]

"""
Explanation: http://bbs.tianya.cn/list.jsp?item=free&nextid=0&order=8&k=PX
通过分析帖子列表的源代码，使用inspect方法，会发现所有要解析的内容都在‘td’这个层级下
End of explanation
"""


def crawler(page_num, file_name):
    try:
        # open the browser
        url = "http://bbs.tianya.cn/list.jsp?item=free&nextid=%d&order=8&k=PX" % page_num
        content = urllib2.urlopen(url).read() #获取网页的html文本
        soup = BeautifulSoup(content, "lxml") 
        articles = soup.find_all('tr')
        # write down info
        for i in articles[1:]:
            td = i.find_all('td')
            title = td[0].text.strip()
            title_url = td[0].a['href']
            author = td[1].text
            author_url = td[1].a['href']
            views = td[2].text
            replies = td[3].text
            date = td[4]['title']
            record = title + '\t' + title_url+ '\t' + author + '\t'+ \
                        author_url + '\t' + views+ '\t'  + replies+ '\t'+ date
            with open(file_name,'a') as p: # '''Note'''：Ａppend mode, run only once!
                        p.write(record.encode('utf-8')+"\n") ##!!encode here to utf-8 to avoid encoding

    except Exception, e:
        print e
        pass

# crawl all pages
for page_num in range(10):
    print (page_num)
    crawler(page_num, '/Users/chengjun/github/cjc2016/data/tianya_bbs_threads_list.txt') 

import pandas as pd

df = pd.read_csv('/Users/chengjun/github/cjc2016/data/tianya_bbs_threads_list.txt', sep = "\t", header=None)
df[:2]

len(df)

df=df.rename(columns = {0:'title', 1:'link', 2:'author',3:'author_page', 4:'click', 5:'reply', 6:'time'})
df[:2]

len(df.link)

"""
Explanation: 抓取天涯论坛PX帖子列表
回帖网络（Thread network）的结构
- 列表
- 主帖
- 回帖
End of explanation
"""


df.author_page[:5]

"""
Explanation: 抓取作者信息
End of explanation
"""


user_info

# user_info = soup.find('div',  {'class', 'userinfo'})('p')
# user_infos = [i.get_text()[4:] for i in user_info]
            
def author_crawler(url, file_name):
    try:
        content = urllib2.urlopen(url).read() #获取网页的html文本
        soup = BeautifulSoup(content, "lxml")
        link_info = soup.find_all('div', {'class', 'link-box'})
        followed_num, fans_num = [i.a.text for i in link_info]
        try:
            activity = soup.find_all('span', {'class', 'subtitle'})
            post_num, reply_num = [j.text[2:] for i in activity[:1] for j in i('a')]
        except:
            post_num, reply_num = 1, 0
        record =  '\t'.join([url, followed_num, fans_num, post_num, reply_num])
        with open(file_name,'a') as p: # '''Note'''：Ａppend mode, run only once!
                    p.write(record.encode('utf-8')+"\n") ##!!encode here to utf-8 to avoid encoding

    except Exception, e:
        print e, url
        record =  '\t'.join([url, 'na', 'na', 'na', 'na'])
        with open(file_name,'a') as p: # '''Note'''：Ａppend mode, run only once!
                    p.write(record.encode('utf-8')+"\n") ##!!encode here to utf-8 to avoid encoding
        pass

for k, url in enumerate(df.author_page):
    if k % 10==0:
        print k
    author_crawler(url, '/Users/chengjun/github/cjc2016/data/tianya_bbs_threads_author_info.txt') 

url = df.author_page[1]
content = urllib2.urlopen(url).read() #获取网页的html文本
soup1 = BeautifulSoup(content, "lxml") 

user_info = soup.find('div',  {'class', 'userinfo'})('p')
area, nid, freq_use, last_login_time, reg_time = [i.get_text()[4:] for i in user_info]
print area, nid, freq_use, last_login_time, reg_time 

link_info = soup1.find_all('div', {'class', 'link-box'})
followed_num, fans_num = [i.a.text for i in link_info]
print followed_num, fans_num

activity = soup1.find_all('span', {'class', 'subtitle'})
post_num, reply_num = [j.text[2:] for i in activity[:1] for j in i('a')]
print post_num, reply_num

print activity[2]

link_info = soup.find_all('div', {'class', 'link-box'})
followed_num, fans_num = [i.a.text for i in link_info]
print followed_num, fans_num

link_info[0].a.text

"""
Explanation: http://www.tianya.cn/62237033
http://www.tianya.cn/67896263
End of explanation
"""


df.link[2]

url = 'http://bbs.tianya.cn' + df.link[2]
url

from IPython.display import display_html, HTML
HTML('<iframe src=http://bbs.tianya.cn/post-free-2848797-1.shtml width=1000 height=500></iframe>')
# the webpage we would like to crawl

post = urllib2.urlopen(url).read() #获取网页的html文本
post_soup = BeautifulSoup(post, "lxml") 
#articles = soup.find_all('tr')

print (post_soup.prettify())[:1000]

pa = post_soup.find_all('div', {'class', 'atl-item'})
len(pa)

print pa[0]

print pa[1]

print pa[89]

"""
Explanation: http://www.tianya.cn/50499450/follow  
还可抓取他们的关注列表和粉丝列表


数据抓取：

使用Python抓取回帖



王成军
wangchengjun@nju.edu.cn
计算传播网 http://computational-communication.com
End of explanation
"""


print pa[0].find('div', {'class', 'bbs-content'}).text.strip()

print pa[87].find('div', {'class', 'bbs-content'}).text.strip()

pa[1].a

print pa[0].find('a', class_ = 'reportme a-link')

print pa[0].find('a', class_ = 'reportme a-link')['replytime']

print pa[0].find('a', class_ = 'reportme a-link')['author']

for i in pa[:10]:
    p_info = i.find('a', class_ = 'reportme a-link')
    p_time = p_info['replytime']
    p_author_id = p_info['authorid']
    p_author_name = p_info['author']
    p_content = i.find('div', {'class', 'bbs-content'}).text.strip()
    p_content = p_content.replace('\t', '')
    print p_time, '--->', p_author_id, '--->', p_author_name,'--->', p_content, '\n'

"""
Explanation: 作者：柠檬在追逐 时间：2012-10-28 21:33:55
@lice5 2012-10-28 20:37:17
作为宁波人 还是说一句：革命尚未成功 同志仍需努力 
-----------------------------
对 现在说成功还太乐观，就怕说一套做一套
作者：lice5 时间：2012-10-28 20:37:17
作为宁波人 还是说一句：革命尚未成功 同志仍需努力
4   /post-free-4242156-1.shtml  2014-04-09 15:55:35 61943225    野渡自渡人   @Y雷政府34楼2014-04-0422:30:34　　野渡君雄文！支持是必须的。　　-----------------------------　　@清坪过客16楼2014-04-0804:09:48　　绝对的权力导致绝对的腐败！　　-----------------------------　　@T大漠鱼T35楼2014-04-0810:17:27　　@周丕东@普欣@拾月霜寒2012@小摸包@姚文嚼字@四號@凌宸@乔志峰@野渡自渡人@曾兵2010@缠绕夜色@曾颖@风青扬请关注
End of explanation
"""


post_soup.find('div', {'class', 'atl-pages'})#['onsubmit']

post_pages = post_soup.find('div', {'class', 'atl-pages'})
post_pages = post_pages.form['onsubmit'].split(',')[-1].split(')')[0]
post_pages

url = 'http://bbs.tianya.cn' + df.link[2]
url_base = ''.join(url.split('-')[:-1]) + '-%d.shtml'
url_base

def parsePage(pa):
    records = []
    for i in pa:
        p_info = i.find('a', class_ = 'reportme a-link')
        p_time = p_info['replytime']
        p_author_id = p_info['authorid']
        p_author_name = p_info['author']
        p_content = i.find('div', {'class', 'bbs-content'}).text.strip()
        p_content = p_content.replace('\t', '').replace('\n', '')#.replace(' ', '')
        record = p_time + '\t' + p_author_id+ '\t' + p_author_name + '\t'+ p_content
        records.append(record)
    return records

import sys
def flushPrint(s):
    sys.stdout.write('\r')
    sys.stdout.write('%s' % s)
    sys.stdout.flush()

url_1 = 'http://bbs.tianya.cn' + df.link[10]
content = urllib2.urlopen(url_1).read() #获取网页的html文本
post_soup = BeautifulSoup(content, "lxml") 
pa = post_soup.find_all('div', {'class', 'atl-item'})
b = post_soup.find('div', class_= 'atl-pages')
b

url_1 = 'http://bbs.tianya.cn' + df.link[0]
content = urllib2.urlopen(url_1).read() #获取网页的html文本
post_soup = BeautifulSoup(content, "lxml") 
pa = post_soup.find_all('div', {'class', 'atl-item'})
a = post_soup.find('div', {'class', 'atl-pages'})
a

a.form

if b.form:
    print 'true'
else:
    print 'false'

import random
import time

def crawler(url, file_name):
    try:
        # open the browser
        url_1 = 'http://bbs.tianya.cn' + url
        content = urllib2.urlopen(url_1).read() #获取网页的html文本
        post_soup = BeautifulSoup(content, "lxml") 
        # how many pages in a post
        post_form = post_soup.find('div', {'class', 'atl-pages'})
        if post_form.form:
            post_pages = post_form.form['onsubmit'].split(',')[-1].split(')')[0]
            post_pages = int(post_pages)
            url_base = '-'.join(url_1.split('-')[:-1]) + '-%d.shtml'
        else:
            post_pages = 1
        # for the first page
        pa = post_soup.find_all('div', {'class', 'atl-item'})
        records = parsePage(pa)
        with open(file_name,'a') as p: # '''Note'''：Ａppend mode, run only once!
            for record in records:    
                p.write('1'+ '\t' + url + '\t' + record.encode('utf-8')+"\n") 
        # for the 2nd+ pages
        if post_pages > 1:
            for page_num in range(2, post_pages+1):
                time.sleep(random.random())
                flushPrint(page_num)
                url2 =url_base  % page_num
                content = urllib2.urlopen(url2).read() #获取网页的html文本
                post_soup = BeautifulSoup(content, "lxml") 
                pa = post_soup.find_all('div', {'class', 'atl-item'})
                records = parsePage(pa)
                with open(file_name,'a') as p: # '''Note'''：Ａppend mode, run only once!
                    for record in records:    
                        p.write(str(page_num) + '\t' +url + '\t' + record.encode('utf-8')+"\n") 
        else:
            pass
    except Exception, e:
        print e
        pass

"""
Explanation: 如何翻页
http://bbs.tianya.cn/post-free-2848797-1.shtml
http://bbs.tianya.cn/post-free-2848797-2.shtml

    http://bbs.tianya.cn/post-free-2848797-3.shtml
End of explanation
"""


url = 'http://bbs.tianya.cn' + df.link[2]
file_name = '/Users/chengjun/github/cjc2016/data/tianya_bbs_threads_test.txt'
crawler(url, file_name)

"""
Explanation: 测试
End of explanation
"""


for k, link in enumerate(df.link):
    flushPrint(link)
    if k % 10== 0:
        print 'This it the post of : ' + str(k)
    file_name = '/Users/chengjun/github/cjc2016/data/tianya_bbs_threads_network.txt'
    crawler(link, file_name)

"""
Explanation: 正式抓取！
End of explanation
"""


dtt = []
with open('/Users/chengjun/github/cjc2016/data/tianya_bbs_threads_network.txt', 'r') as f:
    for line in f:
        pnum, link, time, author_id, author, content = line.replace('\n', '').split('\t')
        dtt.append([pnum, link, time, author_id, author, content])
len(dtt)

dt = pd.DataFrame(dtt)
dt[:5]

dt=dt.rename(columns = {0:'page_num', 1:'link', 2:'time', 3:'author',4:'author_name', 5:'reply'})
dt[:5]

dt.reply[:100]

"""
Explanation: 读取数据
End of explanation
"""


18459/50

"""
Explanation: 总帖数是多少？
http://search.tianya.cn/bbs?q=PX  共有18459 条内容
End of explanation
"""


