
'''
step 1
get the userID and their locations
put them all into a database
'''

from bs4 import BeautifulSoup
import urllib
import sqlite3
from selenium import webdriver
import time
import re
from urllib import request
import random
import pickle
import os
import pytesseract





url_dog = "https://www.douban.com/group/lovelydog/members?start="

url_cat = "https://www.douban.com/group/cat/members?start="

'''
cat = 1 ~ 336770
dog = 1 ~ 156240
            '''
class getInfo(object):

    memberList = []
    type = None
    url = None
    memberNumber = 0
    conn = None
    cursor = None

    def __init__(self, type):
        getInfo.type = type

        if type == "cat":
            getInfo.url = url_cat
            getInfo.memberNumber = 336770
        else:
            getInfo.url = url_dog
            getInfo.memberNumber = 156240

        dbName = "CDPeopleDB.sqlite"

        #iniate the start point
        if not os.path.isfile('stopPoint.pickle'):
            with open('stopPoint.pickle', 'rb') as file:
                pickle.dump(1, file)

        conn = sqlite3.connect(dbName)
        getInfo.conn = conn

        getInfo.cursor = getInfo.conn.cursor()


        # if getInfo.type == 'dog':
        #     getInfo.cursor.execute("drop table if exists DogPeople")
        #     getInfo.cursor.execute("create table DogPeople(id varchar(48), location varchar(48))")
        # else:
        #     getInfo.cursor.execute("drop table if exists CatPeople")
        #     getInfo.cursor.execute("create table CatPeople(id varchar(48), location varchar(48))")


    def sliceContent(self, pageContent):
        pageContent = re.sub(r"<ul>(.*)</ul>", "\\1", pageContent.replace("\n", ""))
        # print(pageContent)
        memberList = re.sub(r'<li class=""> (.*?) </li>', "\\1mark", pageContent.strip())
        memberList = re.split(r"mark", memberList)
        inforContent = re.findall(r'<div class="name">(.*?)</div>', memberList[35])
        for member in memberList:
            if member.strip() != '':
                inforContent = re.findall(r'<div class="name">(.*?)</div>', member)
                if len(inforContent)!= 0:
                    inforContent = inforContent[0].strip()
                    identity = re.findall(r'https://www.douban.com/people/(.*?)/', inforContent)[0]
                    if len(identity)!=0:
                        id = identity[0]
                        location = re.findall(r'<span class="pl">\((.*?)\)</span>', inforContent)
                        if len(location) != 0:
                            coordinate = str(location[0])
                        else:
                            coordinate = 'Unknown'
                    else:
                        continue
                if getInfo.type == 'dog':
                    getInfo.cursor.execute("insert into DogPeople values(?, ?)", (id, coordinate))
                else:
                    getInfo.cursor.execute("insert into CatPeople values(?, ?)", (id, coordinate))
        getInfo.conn.commit()






    def crawler(self):

        opener = urllib.request.build_opener(urllib.request.HTTPSHandler)
        header = ("User-Agent",
                  " Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36")
        opener.addheaders = [header]
        driver = webdriver.Chrome()
        driver.get(getInfo.url)
        time.sleep(20)

        #store the current position in case there is something wrong with the crawlering
        with open('stopPoint.pickle', 'rb') as file:
            startPoint = pickle.load(file)

        #use the record to be the start position
        for i in range(startPoint, getInfo.memberNumber, 35):
            driver.get(getInfo.url+str(i))
            page = driver.page_source
            soup = BeautifulSoup(page, "html5lib")
            print(i)
            with open('stopPoint.pickle', 'wb') as file:
                pickle.dump(i, file)
            memberList = soup.find('div', {'class': 'member-list'}).ul
            content = str(memberList)
            getInfo.sliceContent(self, pageContent=content)
            time.sleep(2+random.random())

# info_dog = getInfo("dog")
# info_dog.crawler()
info_cat = getInfo("cat")
info_cat.crawler()


'''
create table CatPeople
as
select distinct * 
from CatPeople_backup
WHERE not location GLOB '*[A-Za-z]*';

pre-processing to delete locations out of China

'''





[((58, 15, 58, 38), 'sqlite3.connect', 'sqlite3.connect', ({(58, 31, 58, 37): 'dbName'}, {}), '(dbName)', False, 'import sqlite3\n'), ((76, 21, 76, 50), 're.split', 're.split', ({(76, 30, 76, 37): '"""mark"""', (76, 39, 76, 49): 'memberList'}, {}), "('mark', memberList)", False, 'import re\n'), ((77, 23, 77, 83), 're.findall', 're.findall', ({(77, 34, 77, 66): '"""<div class="name">(.*?)</div>"""', (77, 68, 77, 82): 'memberList[35]'}, {}), '(\'<div class="name">(.*?)</div>\', memberList[35])', False, 'import re\n'), ((106, 17, 106, 73), 'urllib.request.build_opener', 'urllib.request.build_opener', ({(106, 45, 106, 72): 'urllib.request.HTTPSHandler'}, {}), '(urllib.request.HTTPSHandler)', False, 'import urllib\n'), ((110, 17, 110, 35), 'selenium.webdriver.Chrome', 'webdriver.Chrome', ({}, {}), '()', False, 'from selenium import webdriver\n'), ((112, 8, 112, 22), 'time.sleep', 'time.sleep', ({(112, 19, 112, 21): '(20)'}, {}), '(20)', False, 'import time\n'), ((54, 15, 54, 49), 'os.path.isfile', 'os.path.isfile', ({(54, 30, 54, 48): '"""stopPoint.pickle"""'}, {}), "('stopPoint.pickle')", False, 'import os\n'), ((116, 25, 116, 42), 'pickle.load', 'pickle.load', ({(116, 37, 116, 41): 'file'}, {}), '(file)', False, 'import pickle\n'), ((122, 19, 122, 50), 'bs4.BeautifulSoup', 'BeautifulSoup', ({(122, 33, 122, 37): 'page', (122, 39, 122, 49): '"""html5lib"""'}, {}), "(page, 'html5lib')", False, 'from bs4 import BeautifulSoup\n'), ((56, 16, 56, 36), 'pickle.dump', 'pickle.dump', ({(56, 28, 56, 29): '(1)', (56, 31, 56, 35): 'file'}, {}), '(1, file)', False, 'import pickle\n'), ((80, 31, 80, 83), 're.findall', 're.findall', ({(80, 42, 80, 74): '"""<div class="name">(.*?)</div>"""', (80, 76, 80, 82): 'member'}, {}), '(\'<div class="name">(.*?)</div>\', member)', False, 'import re\n'), ((125, 16, 125, 36), 'pickle.dump', 'pickle.dump', ({(125, 28, 125, 29): 'i', (125, 31, 125, 35): 'file'}, {}), '(i, file)', False, 'import pickle\n'), ((129, 25, 129, 40), 'random.random', 'random.random', ({}, {}), '()', False, 'import random\n'), ((83, 31, 83, 96), 're.findall', 're.findall', ({(83, 42, 83, 81): '"""https://www.douban.com/people/(.*?)/"""', (83, 83, 83, 95): 'inforContent'}, {}), "('https://www.douban.com/people/(.*?)/', inforContent)", False, 'import re\n'), ((86, 35, 86, 97), 're.findall', 're.findall', ({(86, 46, 86, 82): '"""<span class="pl">\\\\((.*?)\\\\)</span>"""', (86, 84, 86, 96): 'inforContent'}, {}), '(\'<span class="pl">\\\\((.*?)\\\\)</span>\', inforContent)', False, 'import re\n')]