text: #! /usr/local/bin/python3
  # coding: utf-8
  # __author__ = "lsg"
  # __date__ = 2017/10/16 16:11
  
  #加载内置包
  import requests
  import json
  import time
  import sys
  #加载第三方包
  import pandas
  from selenium import webdriver
  from requests.exceptions import RequestException
  from selenium.common.exceptions import WebDriverException
  #加载自己编写的文件
  import qqlist
  import settings
  import transCoordinateSystem
  import mail
  import remote_control
  
  class CookieException(Exception):
      def __init__(self):
          Exception.__init__(self)
  
  class easygospider():
      #初始化基本量
      def __init__(self):
          self.qq_number_list = []
          self.cookie = None
          #输入文件的名称
          self.input = settings.xy_name
          #文件保存的路径
          self.filepath = settings.filepath
          self.filename = settings.filename
          self.qq_number_list = qqlist.qq_list
          self.i=0
  
      # 初始化用于爬虫的网格，形成url
      def initial_paramslist(self):
          """
          :return: List[]
          """
          #filename文件中需要存储的为Wgs84坐标系下渔网为2.5Km的中心点的坐标，格式如data.txt
          #读取渔网中心点
          center = []
          with open(self.input, 'r', encoding='utf-8') as f:
              for item in f.readlines()[1:]:
                  center.append(tuple(item.strip().split(",")[-2:]))
          #生成四至范围（按照2.6km范围生成，防止有遗漏点，如有重复，最后去重的时候处理）
          round = []
          for item in center:
              lng, lat = item
              lng, lat = float(lng), float(lat)
              round.append([lng - 0.5 * settings.lng_delta,
                            lng + 0.5 * settings.lng_delta,
                            lat - 0.5 * settings.lat_delta,
                            lat + 0.5 * settings.lat_delta])
          #生成待抓取的params
          paramslist = []
          for item in round:
              lng_min, lng_max, lat_min, lat_max = item
              lng_min, lat_min = transCoordinateSystem.wgs84_to_gcj02(lng_min, lat_min)
              lng_max, lat_max = transCoordinateSystem.wgs84_to_gcj02(lng_max, lat_max)
              params = {"lng_min": lng_min,
                        "lat_max": lat_max,
                        "lng_max": lng_max,
                        "lat_min": lat_min,
                        "level": 16,
                        "city": "厦门",
                        "lat": "undefined",
                        "lng": "undefined",
                        "_token": ""}
              paramslist.append(params)
          return paramslist
  
      def get_cookie(self):
          j=len(self.qq_number_list)
          while True:
              try:
                  chrome_login = webdriver.Chrome(executable_path="chromedriver.exe")
                  chrome_login.implicitly_wait(10)
                  chrome_login.get("http://c.easygo.qq.com/eg_toc/map.html?origin=csfw&cityid=110000")
                  if self.i>=j:
                      self.i=0
                      qq_ = self.qq_number_list[self.i]
                  else:
                      qq_ = self.qq_number_list[self.i]
                      self.i+=1
                  qq_num = qq_[0]
                  qq_passwd = qq_[1]
                  chrome_login.find_element_by_id("u").send_keys(qq_num)
                  chrome_login.find_element_by_id("p").send_keys(qq_passwd)
                  chrome_login.find_element_by_id("go").click()
                  #检查是否存在验证码
                  time.sleep(5)
                  if "安全验证" in chrome_login.page_source:
                      if settings.CAPTCHA_RECOGNIZ:
                          input('等待手动去除验证码')
                      else:
                          chrome_login.close()
                          continue
                  if "温馨提示" in chrome_login.page_source:
                      chrome_login.close()
                      continue
                  #获取cookie
                  cookies = chrome_login.get_cookies()
                  chrome_login.quit()
                  user_cookie = {}
                  for cookie in cookies:
                      user_cookie[cookie["name"]] = cookie["value"]
                  return user_cookie
              except WebDriverException as e:
                  pass
              finally:
                  try:
                      chrome_login.close()
                  except Exception:
                      pass
  
      def spyder(self,cookie,params):
          user_header = {
              "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/61.0.3163.100 Safari/537.36",
              "Referer": "http://c.easygo.qq.com/eg_toc/map.html?origin=csfw"
          }
          url = "http://c.easygo.qq.com/api/egc/heatmapdata"
          while True:
              try:
                  r = requests.get(url, headers=user_header,
                                   cookies=cookie, params=params)
                  # print(r.status_code)
                  if r.status_code == 200:
                      return r.text
                  else:
                      raise CookieException
              except RequestException:
                  self.spyder(cookie, params)
  
      def save(self,text,time_now,file_name):
          try:
              with open(file_name, 'r') as f:
                  f.readline()
          except FileNotFoundError as e:
              with open(file_name, 'w', encoding='utf-8') as f:
                  f.write('count,wgs_lng,wgs_lat,time\n')
          # 写入数据
          with open(file_name, "a", encoding="utf-8") as f:
              if text is None:
                  return
              node_list = json.loads(text)["data"]
              try:
                  min_count = json.loads(text)['max_data']/40
                  for i in node_list:
                      # 此处的算法在宜出行网页后台的js可以找到，文件路径是http://c.easygo.qq.com/eg_toc/js/map-55f0ea7694.bundle.js
                      i['count'] = i['count'] / min_count
                      gcj_lng = 1e-6 * (250.0 * i['grid_x'] + 125.0)
                      gcj_lat = 1e-6 * (250.0 * i['grid_y'] + 125.0)
                      lng, lat = transCoordinateSystem.gcj02_to_wgs84(gcj_lng, gcj_lat)
                      f.write(str(i['count']) + "," + str(lng) + "," + str(lat) + "," + time_now + "\n")
              except IndexError as e:
                  pass
                  # print("此区域没有点信息")
              except TypeError as e:
                  print(node_list)
                  raise CookieException
      def remove_duplicate(self,filepath):
          # df = pandas.read_csv(filepath,sep=",")
          df = pandas.read_csv(filepath, sep=',')
          df = df.drop_duplicates()
          csv_name = filepath.replace(".txt", "去重结果.csv")
          df.to_csv(csv_name,index=False)
  
      def exec(self):
          while True:
              time_now = time.time()
              time_now_str = time.strftime('%Y-%m-%d-%H-%M-%S', time.localtime(time_now))
              write_log("此轮抓取开始")
              cookie = self.get_cookie()
              i = 1
              params_list = self.initial_paramslist()
              for params in params_list:
  
                  #这部分负责每个qq号码抓取的次数
                  if i % settings.fre == 0:
                      cookie = self.get_cookie()
                  while True:
                      try:
                          text = self.spyder(cookie, params)
                          self.save(text, time_now_str, file_name=self.filepath + self.filename + time_now_str + ".txt")
                          break
                      except Exception as e:
                          print(e)
                          cookie = self.get_cookie()
              view_bar(i, len(params_list))
              i += 1
              write_log("此轮抓取完成，开始去重")
              self.remove_duplicate(self.filepath + self.filename + time_now_str + ".txt")
              write_log("去重完成,等待下一轮开始")
              remote_control.update_github()
              write_log("文件已上传")
              mail.send_mail("去重完成,文件已上传,等待下一轮开始")
              time.sleep(settings.sleeptime - int(time.time() - time_now))
  
  def write_log(content):
      with open("log.log",'a',encoding='utf-8') as f:
          info = time.strftime("%Y-%m-%d %H:%M:%S",time.localtime())+content+"\n"
          sys.stdout.write(info)
          f.write(info)
  
  def view_bar(num, total):
      rate = float(num) / float(total)
      rate_num = int(rate * 100)
      r = '\r[%s%s]%d%%' % ("="*(rate_num), " "*(100-rate_num), rate_num, )
      sys.stdout.write(r)
  
  if __name__ == "__main__":
      app = easygospider()
      app.exec()
  
  