#!/usr/bin/env python
# -*- coding: utf-8 -*-

from fake_useragent import UserAgent
from bs4 import BeautifulSoup
import requests
import MySQLdb
import random
import socket
import struct
import threading
import Queue
import time

host = 'localhost'
user = 'mariadb'
password = 'mariadb'
database = 'ip'
addsql = ('insert into nnip(ip, port, protocol) values(inet_aton(%s), %s, %s)'
            ' on duplicate key update '
            'ip=values(ip), port=values(port), protocol=values(protocol)')
delsql = 'delete from nnip where ip = inet_aton(%s)'
ua = UserAgent()

class DB:
    def __init__(self):
        self.conn = MySQLdb.connect(host, user, password, database)
        self.conn.autocommit(False)
        self.cursor = self.conn.cursor()

    def add(self, ip, port, protocol):
        self.cursor.execute(addsql, (ip, port, protocol))

    def addmany(self, proxies):
        self.cursor.executemany(addsql, proxies)

    def delete(self, ip):
        self.cursor.execute(delsql, (ip,))

    def __del__(self):
        self.conn.commit()
        self.cursor.close()
        self.conn.close()

def check(qin, qout):
    proxy = {}
    url = 'http://www.icanhazip.com'
    while True:
        (ip, port, protocol) = qin.get()
        header = {'user-agent': ua.random}
        proxy['http'] = protocol + '://' + ip + ':' + port
        proxy['https'] = protocol + '://' + ip + ':' + port
        try:
            r = requests.get(url, headers=header, proxies=proxy, timeout=30)
            if r.status_code == requests.codes.ok:
                qout.put((ip, port, protocol))
        except:
            pass
        qin.task_done()

def getproxy(proxies, q):
    proxy = {}
    try:
        (ip, port, protocol) = q.get(timeout=5)
        q.task_done()
        proxy['http'] = protocol + '://' + ip + ':' + port
        proxy['https'] = protocol + '://' + ip + ':' + port
        proxies.append((ip, port, protocol))
    except:
        pass
    return proxy

def getxici(proxies, qin, qout):
    s = 'http://www.xicidaili.com/nn/'
    headers = {'User-Agent': ua.chrome}
    proxy = {}
    times = 0

    session = requests.Session()
    x = 1
    while x <= 1168:
        times += 1
        if times % 200 == 0:
            session.close()
            time.sleep(1)
            session = requests.Session()
            proxy = getproxy(proxies, qout)
            headers['Referer'] = None
            times = 0

        url = s + str(x)
        try:
            r = session.get(url, headers=headers, proxies=proxy, timeout=30)
            headers['Referer'] = r.url
            soup = BeautifulSoup(r.content, 'lxml')
            table = soup.find('table')
            for row in table.find_all('tr'):
                data = []
                for col in row.find_all('td'):
                    data.append(col.find(text=True))
                if data != []:
                    if data[5] == 'socks4/5':
                        data[5] = 'socks5'
                    qin.put((data[1], data[2], data[5]))
            x += 1
            time.sleep(1)
        except Exception as e:
            session.close()
            time.sleep(1)
            session = requests.Session()
            proxy = getproxy(proxies, qout)
            headers['Referer'] = None
            times = 0

def getk(qin, qout):
    s = 'http://www.kuaidaili.com/free/inha/'
    headers = {'User-Agent': ua.chrome}

    with requests.Session() as session:
        x = 1
        while x <= 1391:
            url = s + str(x)
            try:
                r = session.get(url, headers=headers, timeout=30)
                headers['Referer'] = r.url
                soup = BeautifulSoup(r.content, 'lxml')
                table = soup.find('table')
                for row in table.find_all('tr'):
                    data = []
                    for col in row.find_all('td'):
                        data.append(col.find(text=True))
                    if data != []:
                        qin.put((data[0], data[1], data[3]))
                x += 1
                time.sleep(1)
            except Exception as e:
                pass

def main():
    proxies = []
    qin = Queue.Queue()
    qout = Queue.Queue()

    for i in range(1024):
        t = threading.Thread(target=check, args=(qin, qout))
        t.daemon = True
        t.start()

    t1 = threading.Thread(target=getxici, args=(proxies, qin, qout))
    t2 = threading.Thread(target=getk, args=(qin, qout))
    for t in t1,t2:
        t.start()
    for t in t1,t2:
        t.join()

    qin.join()

    while True:
        try:
            (ip, port, protocol) = qout.get_nowait()
            proxies.append((ip, port, protocol))
        except:
            break

    db = DB()
    db.addmany(proxies)
    del db

if __name__ == '__main__':
    main()

