<reponame>txgyy/ipproxytool<gh_stars>10-100
# -*- coding=utf-8 -*-

import datetime
import json
import random
import re
import time
import config

from scrapy import Request
from scrapy.selector import Selector
from ipproxytool.spiders.validator.validator import Validator


class AJKSpider(Validator):
    name = 'ajk'

    concurrent_requests = 16

    def __init__(self, name = None, **kwargs):
        super(AJKSpider, self).__init__(name, **kwargs)

        self.region_urls = [
            'aolinpikegongyuan/',
            'anzhen/',
            'baiziwan/',
            'beiyuan/',
            'balizhuangb/',
            'chaoyanggongyuandong/',
            'chaowaidajie/',
            'changying/',
            'chaoyangjichang/',
            'chaoqing/',
            'chaoyanggongyuanxi/',
            'dawanglu/',
            'dongbaa/',
            'dougezhuang/',
            'dongdaqiao/',
            'dingfuzhuang/',
            'fatou/',
            'gaobeidian/',
            'guanzhuang/',
            'guomao/',
            'hepinglibei/',
            'huaweiqiaocy/',
            'jinsongdong/',
            'jianzhanxiang/',
            'jianxiangqiao/',
            'jianguomenwai/',
            'jiuxianqiao/',
            'jinsongxi/',
            'laiguangying/',
            'liufang/',
            'nanshatan/',
            'panjiayuan/',
            'shilihe/',
            'sanlitun/',
            'sihui/',
            'shuangqiaoc/',
            'shifoying/',
            'shibalidian/',
            'shaoyaoju/',
            'shuangjing/',
            'sanyuanqiao/',
            'taiyanggong/',
            'tuanjiehu/',
            'wangjingxi/',
            'wangjingdong/',
            'xiaohongmen/',
            'yayuncun/',
            'chaoyang/',
            'haidian/',
            'dongchenga/',
            'xicheng/',
            'fengtai/',
            'tongzhou/',
            'shijingshan/',
            'changping/',
            'daxing/',
            'shunyi/',
            'fangshan/',
            'mentougou/',
            'miyun/',
            'huairou/',
            'pinggua/',
            'yanqing/',
            'beijingzhoubiana/',
            'baishiqiao/',
            'chedaogou/',
            'dinghuisi/',
            'erlizhuang/',
            'gongzhufenxi/',
            'ganjiakou/',
            'gongzhufendong/',
            'haidianbeibu/',
            'junbo/',
            'madians/',
            'malianwa/',
            'mudanyuan/',
            'qinghe/',
            'shijicheng/',
            'sijiqing/',
            'suzhouqiao/',
            'shangdi/',
            'shuangyushu/',
            'tiancun/',
            'wenquand/',
            'wanquanhe/',
            'wanshoulu/',
            'wanliu/',
            'wudaokou/',
            'weigongcun/',
            'xiangshandong/',
            'xibeiwang/',
            'xierqi/',
            'xiangshangxi/',
            'xiaoxitian/',
            'xisanqi/',
            'xueyuanlu/',
            'yuquanlu/',
            'yiheyuan/',
            'yuanmingyuan/',
            'zaojunmiao/',
            'zizhuqiao/',
            'zhichunlu/',
            'zhongguancun/',
            'andingmen/',
            'chongwenmens/',
            'chaoyangmennei/',
            'dongzhimenwai/',
            'donghuashis/',
            'dongdan/',
            'dongsia/',
            'dongzhimennei/',
            'dengshikou/',
            'guangqumen/',
            'hepinglianan/',
            'jiaodaokou/',
            'jianguomennei/',
            'longtanhus/',
            'qianmens/',
            'tiantans/',
            'wangfujing/',
            'yongdingmens/',
            'yonghegong/',
            'baizhifangs/',
            'baiyunluxc/',
            'changchunjiexc/',
            'chegongzhuanga/',
            'deshengmen/',
            'fuchengmen/',
            'guanganmenwai/',
            'guanganmennei/',
            'guanyuan/',
            'hepingmen/',
            'jinrongjie/',
            'liupukang/',
            'maliandaos/',
            'shichahai/',
            'tianningshi/',
            'taorantings/',
            'xuanwumens/',
            'xizhimenwai/',
            'xisi/',
            'xizhimen/',
            'xinjiekou/',
            'xidan/',
            'yuetan/',
            'beidadi/',
            'caoqiao/',
            'chengshousi/',
            'caihuying/',
            'dahongmen/',
            'fangzhuang/',
            'heyi/',
            'jiaomen/',
            'kandanqiao/',
            'kejiyuanquft/',
            'lizeqiao/',
            'liuliqiaoxi/',
            'lugouqiao/',
            'liujiayao/',
            'liuliqiaodong/',
            'majiabao/',
            'muxiyuan/',
            'puhuangyu/',
            'qilizhuang/',
            'qingta/',
            'songjiazhuang/',
            'xinfadi/',
            'xiluoyuan/',
            'youanmenwai/',
            'yuquanying/',
            'beiguan/',
            'guoyuan/',
            'jiukeshu/',
            'luyuan/',
            'liyuan/',
            'majuqiao/',
            'qiaozhuang/',
            'tuqiao/',
            'tongzhouquqita/',
            'tongzhoubeiyuan/',
            'wuyihuayuan/',
            'xinhuadajie/',
            'bajiao/',
            'gucheng/',
            'laoshan/',
            'lugu/',
            'pingguoyuan/',
            'shijingshana/',
            'yuquanluxi/',
            'yangzhuang/',
            'baishanzhen/',
            'beiqijia/',
            'changpingquqita/',
            'changpingxiancheng/',
            'dongxiaokouzhen/',
            'huoying/',
            'huilongguan/',
            'longze/',
            'lishuiqiao/',
            'nanshao/',
            'nankou/',
            'shahea/',
            'tiantongyuan/',
            'xingshouzhen/',
            'xiaotangshanbei/',
            'xiaotangshannan/',
            'zhuxinzhuang/',
            'daxingquqita/',
            'guanyinsi/',
            'gaomidian/',
            'huangcun/',
            'jiugong/',
            'luchengxiang/',
            'panggezhuang/',
            'qingyundianzhen/',
            'tiangongyuannan/',
            'tiangongyuanbei/',
            'xihongmen/',
            'yinghaizhen/',
            'yizhuang/',
            'zaoyuans/',
            'houshayu/',
            'jichangfujin/',
            'liqiao/',
            'mapo/',
            'shunyiquqita/',
            'shunyicheng/',
            'tianzhu/',
            'yangzhen/',
            'zhongyangbieshuqu/',
            'changyang/',
            'chengguanbj/',
            'doudian/',
            'fangshanquqita/',
            'guandaozhen/',
            'hancunhe/',
            'liangxiang/',
            'liulihe/',
            'yancun/',
            'yanshan/',
            'binhexiqu/',
            'chengzi/',
            'dayu/',
            'fengcun/',
            'mentougouquqita/',
            'shimenying/',
            'yongdingzhen/',
            'badaling/',
            'dayushu/',
            'kangzhuang/',
            'yanqingquqita/',
            'yanqingchengqu/',
            'baodinga/',
            'langfanga/',
            'qinhuangdaoa/',
            'tangshang/',
            'weihaia/',
            'yantaia/',
            'yanjiao/',
            'zhangjiakou/',
        ]

        self.price_urls = [
            'zj5332/',
            'zj297/',
            'zj298/',
            'zj299/',
            'zj300/',
            'zj301/',
            'zj33/',
            'zj5333/',
            'zj5334/',
            'zj5335/',
            'zj5336/',
        ]

        self.init()

    def start_requests(self):
        count = self.sql.get_proxy_count(self.name)
        count_free = self.sql.get_proxy_count(config.httpbin_table)

        ids = self.sql.get_proxy_ids(self.name)
        ids_httpbin = self.sql.get_proxy_ids(config.httpbin_table)

        for i in range(0, count + count_free):
            table = self.name if (i < count) else config.httpbin_table
            id = ids[i] if i < count else ids_httpbin[i - len(ids)]

            proxy = self.sql.get_proxy_with_id(table, id)
            if proxy == None:
                continue

            full_url = 'https://bj.zu.anjuke.com/fangyuan/{region}p{page}-{price}'.format(
                region = random.choice(self.region_urls), price = random.choice(self.price_urls), page = 1)
            cur_time = time.time()
            yield Request(
                url = full_url,
                headers = {
                    'user-agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36'
                },
                dont_filter = True,
                meta = {
                    'cur_time': cur_time,
                    'download_timeout': self.timeout,
                    'proxy_info': proxy,
                    'table': table,
                    'proxy': 'http://%s:%s' % (proxy.ip, proxy.port),
                },
                callback = self.success_parse,
                errback = self.error_parse,
            )
