text: # -*- coding: utf-8 -*-
  # !/usr/bin/env python
  """
  -------------------------------------------------
     File Name：     GetFreeProxy.py
     Description :  抓取免费代理
     Author :       JHao
     date：          2016/11/25
  -------------------------------------------------
     Change Activity:
                     2016/11/25:
  -------------------------------------------------
  """
  import re
  import sys
  import requests
  from time import sleep
  
  sys.path.append('..')
  
  from Util.WebRequest import WebRequest
  from Util.utilFunction import getHtmlTree
  
  # for debug to disable insecureWarning
  requests.packages.urllib3.disable_warnings()
  
  
  class GetFreeProxy(object):
      """
      proxy getter
      """
  
      @staticmethod
      def rsnew2():
          try:
              url = 'http://proxy.link/list/get/9695da9193779627fe05cbdc9843f3a4'
              r = requests.get(url, timeout=(20, 60))
              if r.status_code != 200:
                  return
  
              ip_list = r.text
              if ip_list != "":
                  ip_list = ip_list.split("\n")
                  for _proxy in ip_list:
                      yield _proxy
              else:
                  print('get proxies is empty, check request permission')
          except Exception as e:
              print(e)
  
      @staticmethod
      def freeProxy01():
          """
          无忧代理 http://www.data5u.com/
          几乎没有能用的
          :return:
          """
          url_list = [
              'http://www.data5u.com/',
              'http://www.data5u.com/free/gngn/index.shtml',
              'http://www.data5u.com/free/gnpt/index.shtml'
          ]
          key = 'ABCDEFGHIZ'
          for url in url_list:
              html_tree = getHtmlTree(url)
              ul_list = html_tree.xpath('//ul[@class="l2"]')
              for ul in ul_list:
                  try:
                      ip = ul.xpath('./span[1]/li/text()')[0]
                      classnames = ul.xpath('./span[2]/li/attribute::class')[0]
                      classname = classnames.split(' ')[1]
                      port_sum = 0
                      for c in classname:
                          port_sum *= 10
                          port_sum += key.index(c)
                      port = port_sum >> 3
                      yield '{}:{}'.format(ip, port)
                  except Exception as e:
                      print(e)
  
      @staticmethod
      def freeProxy02(count=20):
          """
          代理66 http://www.66ip.cn/
          :param count: 提取数量
          :return:
          """
          urls = [
              "http://www.66ip.cn/mo.php?sxb=&tqsl={}&port=&export=&ktip=&sxa=&submit=%CC%E1++%C8%A1&textarea=",
              "http://www.66ip.cn/nmtq.php?getnum={}&isp=0&anonymoustype=0&s"
              "tart=&ports=&export=&ipaddress=&area=0&proxytype=2&api=66ip"
          ]
  
          try:
              import execjs
              import requests
  
              headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:34.0) Gecko/20100101 Firefox/34.0',
                         'Accept': '*/*',
                         'Connection': 'keep-alive',
                         'Accept-Language': 'zh-CN,zh;q=0.8'}
              session = requests.session()
              src = session.get("http://www.66ip.cn/", headers=headers).text
              src = src.split("</script>")[0] + '}'
              src = src.replace("<script>", "function test() {")
              src = src.replace("while(z++)try{eval(", ';var num=10;while(z++)try{var tmp=')
              src = src.replace(");break}", ";num--;if(tmp.search('cookie') != -1 | num<0){return tmp}}")
              ctx = execjs.compile(src)
              src = ctx.call("test")
              src = src[src.find("document.cookie="): src.find("};if((")]
              src = src.replace("document.cookie=", "")
              src = "function test() {var window={}; return %s }" % src
              cookie = execjs.compile(src).call('test')
              js_cookie = cookie.split(";")[0].split("=")[-1]
          except Exception as e:
              print(e)
              return
  
          for url in urls:
              try:
                  html = session.get(url.format(count), cookies={"__jsl_clearance": js_cookie}, headers=headers).text
                  ips = re.findall(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}", html)
                  for ip in ips:
                      yield ip.strip()
              except Exception as e:
                  print(e)
                  pass
  
      @staticmethod
      def freeProxy03(page_count=1):
          """
          西刺代理 http://www.xicidaili.com
          :return:
          """
          url_list = [
              'http://www.xicidaili.com/nn/',  # 高匿
              'http://www.xicidaili.com/nt/',  # 透明
          ]
          for each_url in url_list:
              for i in range(1, page_count + 1):
                  page_url = each_url + str(i)
                  tree = getHtmlTree(page_url)
                  proxy_list = tree.xpath('.//table[@id="ip_list"]//tr[position()>1]')
                  for proxy in proxy_list:
                      try:
                          yield ':'.join(proxy.xpath('./td/text()')[0:2])
                      except Exception as e:
                          pass
  
      @staticmethod
      def freeProxy04():
          """
          guobanjia http://www.goubanjia.com/
          :return:
          """
          url = "http://www.goubanjia.com/"
          tree = getHtmlTree(url)
          proxy_list = tree.xpath('//td[@class="ip"]')
          # 此网站有隐藏的数字干扰，或抓取到多余的数字或.符号
          # 需要过滤掉<p style="display:none;">的内容
          xpath_str = """.//*[not(contains(@style, 'display: none'))
                                          and not(contains(@style, 'display:none'))
                                          and not(contains(@class, 'port'))
                                          ]/text()
                                  """
          for each_proxy in proxy_list:
              try:
                  # :符号裸放在td下，其他放在div span p中，先分割找出ip，再找port
                  ip_addr = ''.join(each_proxy.xpath(xpath_str))
  
                  # HTML中的port是随机数，真正的端口编码在class后面的字母中。
                  # 比如这个：
                  # <span class="port CFACE">9054</span>
                  # CFACE解码后对应的是3128。
                  port = 0
                  for _ in each_proxy.xpath(".//span[contains(@class, 'port')]"
                                            "/attribute::class")[0]. \
                          replace("port ", ""):
                      port *= 10
                      port += (ord(_) - ord('A'))
                  port /= 8
  
                  yield '{}:{}'.format(ip_addr, int(port))
              except Exception as e:
                  pass
  
      @staticmethod
      def freeProxy05():
          """
          快代理 https://www.kuaidaili.com
          """
          url_list = [
              'https://www.kuaidaili.com/free/inha/',
              'https://www.kuaidaili.com/free/intr/'
          ]
          for url in url_list:
              tree = getHtmlTree(url)
              proxy_list = tree.xpath('.//table//tr')
              sleep(1)  # 必须sleep 不然第二条请求不到数据
              for tr in proxy_list[1:]:
                  yield ':'.join(tr.xpath('./td/text()')[0:2])
  
      @staticmethod
      def freeProxy06():
          """
          码农代理 https://proxy.coderbusy.com/
          :return:
          """
          urls = ['https://proxy.coderbusy.com/']
          for url in urls:
              tree = getHtmlTree(url)
              proxy_list = tree.xpath('.//table//tr')
              for tr in proxy_list[1:]:
                  yield ':'.join(tr.xpath('./td/text()')[0:2])
  
      @staticmethod
      def freeProxy07():
          """
          云代理 http://www.ip3366.net/free/
          :return:
          """
          urls = ['http://www.ip3366.net/free/?stype=1',
                  "http://www.ip3366.net/free/?stype=2"]
          request = WebRequest()
          for url in urls:
              r = request.get(url, timeout=10)
              proxies = re.findall(r'<td>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\s\S]*?<td>(\d+)</td>', r.text)
              for proxy in proxies:
                  yield ":".join(proxy)
  
      @staticmethod
      def freeProxy08():
          """
          IP海 http://www.iphai.com/free/ng
          :return:
          """
          urls = [
              'http://www.iphai.com/free/ng',
              'http://www.iphai.com/free/np',
              'http://www.iphai.com/free/wg',
              'http://www.iphai.com/free/wp'
          ]
          request = WebRequest()
          for url in urls:
              r = request.get(url, timeout=10)
              proxies = re.findall(r'<td>\s*?(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\s*?</td>[\s\S]*?<td>\s*?(\d+)\s*?</td>',
                                   r.text)
              for proxy in proxies:
                  yield ":".join(proxy)
  
      @staticmethod
      def freeProxy09(page_count=1):
          """
          http://ip.jiangxianli.com/?page=
          免费代理库
          :return:
          """
          for i in range(1, page_count + 1):
              url = 'http://ip.jiangxianli.com/?country=中国&?page={}'.format(i)
              html_tree = getHtmlTree(url)
              for index, tr in enumerate(html_tree.xpath("//table//tr")):
                  if index == 0:
                      continue
                  yield ":".join(tr.xpath("./td/text()")[0:2]).strip()
  
      # @staticmethod
      # def freeProxy10():
      #     """
      #     墙外网站 cn-proxy
      #     :return:
      #     """
      #     urls = ['http://cn-proxy.com/', 'http://cn-proxy.com/archives/218']
      #     request = WebRequest()
      #     for url in urls:
      #         r = request.get(url, timeout=10)
      #         proxies = re.findall(r'<td>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\w\W]<td>(\d+)</td>', r.text)
      #         for proxy in proxies:
      #             yield ':'.join(proxy)
  
      # @staticmethod
      # def freeProxy11():
      #     """
      #     https://proxy-list.org/english/index.php
      #     :return:
      #     """
      #     urls = ['https://proxy-list.org/english/index.php?p=%s' % n for n in range(1, 10)]
      #     request = WebRequest()
      #     import base64
      #     for url in urls:
      #         r = request.get(url, timeout=10)
      #         proxies = re.findall(r"Proxy\('(.*?)'\)", r.text)
      #         for proxy in proxies:
      #             yield base64.b64decode(proxy).decode()
  
      # @staticmethod
      # def freeProxy12():
      #     urls = ['https://list.proxylistplus.com/Fresh-HTTP-Proxy-List-1']
      #     request = WebRequest()
      #     for url in urls:
      #         r = request.get(url, timeout=10)
      #         proxies = re.findall(r'<td>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\s\S]*?<td>(\d+)</td>', r.text)
      #         for proxy in proxies:
      #             yield ':'.join(proxy)
  
      @staticmethod
      def freeProxy13(max_page=2):
          """
          http://www.qydaili.com/free/?action=china&page=1
          齐云代理
          :param max_page:
          :return:
          """
          base_url = 'http://www.qydaili.com/free/?action=china&page='
          request = WebRequest()
          for page in range(1, max_page + 1):
              url = base_url + str(page)
              r = request.get(url, timeout=10)
              proxies = re.findall(
                  r'<td.*?>(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})</td>[\s\S]*?<td.*?>(\d+)</td>',
                  r.text)
              for proxy in proxies:
                  yield ':'.join(proxy)
  
      @staticmethod
      def freeProxy14(max_page=2):
          """
          http://www.89ip.cn/index.html
          89免费代理
          :param max_page:
          :return:
          """
          base_url = 'http://www.89ip.cn/index_{}.html'
          request = WebRequest()
          for page in range(1, max_page + 1):
              url = base_url.format(page)
              r = request.get(url, timeout=10)
              proxies = re.findall(
                  r'<td.*?>[\s\S]*?(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})[\s\S]*?</td>[\s\S]*?<td.*?>[\s\S]*?(\d+)[\s\S]*?</td>',
                  r.text)
              for proxy in proxies:
                  yield ':'.join(proxy)
  
      @staticmethod
      def freeProxy15():
          urls = ['http://www.xiladaili.com/putong/',
                  "http://www.xiladaili.com/gaoni/",
                  "http://www.xiladaili.com/http/",
                  "http://www.xiladaili.com/https/"]
          request = WebRequest()
          for url in urls:
              r = request.get(url, timeout=10)
              ips = re.findall(r"\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}:\d{1,5}", r.text)
              for ip in ips:
                  yield ip.strip()
  
  
  if __name__ == '__main__':
      from CheckProxy import CheckProxy
  
      # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy01)
      # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy02)
      # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy03)
      # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy04)
      # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy05)
      # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy06)
      # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy07)
      # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy08)
      # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy09)
      # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy13)
      # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy14)
      # CheckProxy.checkGetProxyFunc(GetFreeProxy.freeProxy15)
  
      CheckProxy.checkAllGetProxyFunc()
  