text: #!/usr/bin/env python
  # -*- coding:utf-8 -*-
  
  import os
  import re
  import urllib
  import json
  import socket
  import urllib.request
  import urllib.parse
  import urllib.error
  # 设置超时
  import time
  
  timeout = 5
  socket.setdefaulttimeout(timeout)
  
  
  class Crawler:
      # 睡眠时长
      __time_sleep = 0.1
      __amount = 0
      __start_amount = 0
      __counter = 0
      headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:23.0) Gecko/20100101 Firefox/23.0'}
  
      # 获取图片url内容等
      # t 下载图片时间间隔
      def __init__(self, t=0.1):
          self.time_sleep = t
  
      # 获取后缀名
      def get_suffix(self, name):
          m = re.search(r'\.[^\.]*$', name)
          if m.group(0) and len(m.group(0)) <= 5:
              return m.group(0)
          else:
              return '.jpeg'
  
      # 获取referrer，用于生成referrer
      def get_referrer(self, url):
          par = urllib.parse.urlparse(url)
          if par.scheme:
              return par.scheme + '://' + par.netloc
          else:
              return par.netloc
  
          # 保存图片
      def save_image(self, rsp_data, word):
          if not os.path.exists("./" + word):
              os.mkdir("./" + word)
          # 判断名字是否重复，获取图片长度
          self.__counter = len(os.listdir('./' + word)) + 1
          for image_info in rsp_data['imgs']:
  
              try:
                  time.sleep(self.time_sleep)
                  suffix = self.get_suffix(image_info['objURL'])
                  # 指定UA和referrer，减少403
                  refer = self.get_referrer(image_info['objURL'])
                  opener = urllib.request.build_opener()
                  opener.addheaders = [
                      ('User-agent', 'Mozilla/5.0 (Windows NT 10.0; WOW64; rv:55.0) Gecko/20100101 Firefox/55.0'),
                      ('Referer', refer)
                  ]
                  urllib.request.install_opener(opener)
                  # 保存图片
                  urllib.request.urlretrieve(image_info['objURL'], './' + word + '/' + str(self.__counter) + str(suffix))
              except urllib.error.HTTPError as urllib_err:
                  print(urllib_err)
                  continue
              except Exception as err:
                  time.sleep(1)
                  print(err)
                  print("产生未知错误，放弃保存")
                  continue
              else:
                  print("图片+1,已有" + str(self.__counter) + "张图")
                  self.__counter += 1
          return
  
      # 开始获取
      def get_images(self, word='图'):
          search = urllib.parse.quote(word)
          # pn int 图片数
          pn = self.__start_amount
          while pn < self.__amount:
  
              url = 'http://image.baidu.com/search/avatarjson?tn=resultjsonavatarnew&ie=utf-8&word=' + search + '&cg=girl&pn=' + str(
                  pn) + '&rn=60&itg=0&z=0&fr=&width=&height=&lm=-1&ic=0&s=0&st=-1&gsm=1e0000001e'
              #url = 'http://cn.bing.com/images/search/avatarjson?tn=resultjsonavatarnew&ie=utf-8&word=' + search + '&cg=girl&pn=' + str(
             # pn) + '&rn=60&itg=0&z=0&fr=&width=&height=&lm=-1&ic=0&s=0&st=-1&gsm=1e0000001e'
  
              # 设置header防ban
              try:
                  time.sleep(self.time_sleep)
                  req = urllib.request.Request(url=url, headers=self.headers)
                  page = urllib.request.urlopen(req)
                  rsp = page.read().decode('unicode_escape')
              except UnicodeDecodeError as e:
                  print(e)
                  print('-----UnicodeDecodeErrorurl:', url)
              except urllib.error.URLError as e:
                  print(e)
                  print("-----urlErrorurl:", url)
              except socket.timeout as e:
                  print(e)
                  print("-----socket timout:", url)
              else:
                  # 解析json
                  rsp_data = json.loads(rsp)
                  self.save_image(rsp_data, word)
                  # 读取下一页
                  print("下载下一页")
                  pn += 60
              finally:
                  page.close()
          print("下载任务结束")
          return
  
      def start(self, word, spider_page_num=1, start_page=1):
          """
          :param word: 抓取的关键词
          :param spider_page_num: 需要抓取数据页数 总抓取图片数量为 页数x60
          :param start_page:起始页数
  
          """
          self.__start_amount = (start_page - 1) * 60
          self.__amount = spider_page_num * 60 + self.__start_amount
          self.get_images(word)
  
  
  if __name__ == '__main__':
      crawler = Crawler(0.05)  # 抓取延迟为 0.05
      crawler.start('海警船', 8, 1)  # 抓取关键词为 ，总数为 10 页（即总共 10*60=600 张），起始抓取的页码为 1
  
  