<gh_stars>0
import re
from time import sleep
from pandas.core.frame import DataFrame
import scrapy
import json
import pandas

from scrapy import selector


class CompanySpider(scrapy.Spider):
    name = 'company'
    allowed_domains = ['172.16.31.10:8088']
    start_urls = ['http://172.16.31.101:8088/talent/talententerprise/page.do']

    # ?company&currentPageNo=1&pageSize=100

    url = "http://172.16.31.10:8088/talent/talententerprise/page.do"
    company = ''
    total = 0
    currentPageNo = 1
    pageSize = 100
    save_file = 'csv/company/talententerprise.csv'
    body = {'company': '',
            'currentPageNo': str(currentPageNo),
            'pageSize': str(pageSize)
            }
    df = None
    data = []

    def start_requests(self):
        # return super().start_requests()
        self.logger.info(self.body)
        return [scrapy.FormRequest(self.url,
                                   formdata=self.body,
                                   #    method='POST',
                                   callback=self.parse)]

    def logged_in(self, response):
        self.logger.debug(response.text)
        pass

    def parse(self, response):

        try:
            jStr = json.loads(response.text)
            self.logger.debug(
                "request message with {0}, {1}".format(jStr['flag'], jStr['errMsg']))

            if jStr['flag'] == True and jStr['errMsg'] == "success":
                self.total = jStr['data']['totalCount']
                self.currentPageNo = jStr['data']['currentPageNo']

                if len(jStr['data']['result']) > 0:
                    self.logger.debug("add data: {0}".format(
                        len(jStr['data']['result'])))

                    self.data.extend(jStr['data']['result'])

            if self.pageSize * self.currentPageNo < self.total:
                self.currentPageNo += 1
                self.logger.debug("total: {0}".format(self.total))
                self.logger.debug(
                    "request index page {0}".format(self.currentPageNo))
                yield scrapy.FormRequest(self.url,
                                         callback=self.parse,
                                         #  method='POST',
                                         formdata={
                                             'company': '',
                                             'currentPageNo': str(self.currentPageNo),
                                             'pageSize': str(self.pageSize)
                                         }, dont_filter=True)
                #  dont_filter=True)
            else:
                self.df = DataFrame(self.data)
                self.logger.info(
                    "save data to csv format in {0}".format(self.save_file))
                self.df.to_csv(self.save_file,encoding='utf-8')
                self.logger.info("save sucessfuly!")

            # 休眠十秒钟，减少访问频次，防止被禁
            sleep(10)

        except KeyError:
            print(response.text)
