text: # -*- coding: utf-8 -*-
  
  # Scrapy settings for sfish project
  #
  # For simplicity, this file contains only settings considered important or
  # commonly used. You can find more settings consulting the documentation:
  #
  #     http://doc.scrapy.org/en/latest/topics/settings.html
  #     http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html
  #     http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html
  
  BOT_NAME = 'sfish'
  
  SPIDER_MODULES = ['sfish.spiders']
  NEWSPIDER_MODULE = 'sfish.spiders'
  
  # Crawl responsibly by identifying yourself (and your website) on the user-agent
  # USER_AGENT = 'sfish (+http://www.yourdomain.com)'
  
  # Obey robots.txt rules
  ROBOTSTXT_OBEY = False
  # REDIRECT_ENABLED = False
  DOWNLOAD_TIMEOUT = 20
  # RETRY_ENABLED = False
  # CONCURRENT_REQUESTS = 60
  
  # Configure maximum concurrent requests performed by Scrapy (default: 16)
  # CONCURRENT_REQUESTS = 32
  
  # Configure a delay for requests for the same website (default: 0)
  # See http://scrapy.readthedocs.org/en/latest/topics/settings.html#download-delay
  # See also autothrottle settings and docs
  # DOWNLOAD_DELAY = 2
  # The download delay setting will honor only one of:
  # CONCURRENT_REQUESTS_PER_DOMAIN = 16
  # CONCURRENT_REQUESTS_PER_IP = 16
  
  # Disable cookies (enabled by default)
  COOKIES_ENABLED = False
  # DEPTH_LIMIT = 3
  # Disable Telnet Console (enabled by default)
  # TELNETCONSOLE_ENABLED = False
  
  # Override the default request headers:
  # DEFAULT_REQUEST_HEADERS = {
  #   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
  #   'Accept-Language': 'en',
  # }
  
  # Enable or disable spider middlewares
  # See http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html
  # SPIDER_MIDDLEWARES = {
  #    'sfish.middlewares.LittlefishSpiderMiddleware': 543,
  # }
  
  # Enable or disable downloader middlewares
  # See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html
  DOWNLOADER_MIDDLEWARES = {
      'sfish.middlewares.UserAgentMiddleware': 1,
      'sfish.middlewares.ProxyIpMiddleware': 2,
      'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None,
  }
  
  # Enable or disable extensions
  # See http://scrapy.readthedocs.org/en/latest/topics/extensions.html
  # EXTENSIONS = {
  #    'scrapy.extensions.telnet.TelnetConsole': None,
  # }
  
  # Configure item pipelines
  # See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html
  # ITEM_PIPELINES = {
  #     'sfish.pipelines.MmPipeline': 300
  # }
  
  # Enable and configure the AutoThrottle extension (disabled by default)
  # See http://doc.scrapy.org/en/latest/topics/autothrottle.html
  # AUTOTHROTTLE_ENABLED = True
  # The initial download delay
  # AUTOTHROTTLE_START_DELAY = 5
  # The maximum download delay to be set in case of high latencies
  # AUTOTHROTTLE_MAX_DELAY = 60
  # The average number of requests Scrapy should be sending in parallel to
  # each remote server
  # AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
  # Enable showing throttling stats for every response received:
  # AUTOTHROTTLE_DEBUG = False
  
  # Enable and configure HTTP caching (disabled by default)
  # See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
  # HTTPCACHE_ENABLED = True
  # HTTPCACHE_EXPIRATION_SECS = 0
  # HTTPCACHE_DIR = 'httpcache'
  # HTTPCACHE_IGNORE_HTTP_CODES = []
  # HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
  
  import os
  import sys
  
  PROJECT_DIR = os.path.dirname(os.path.abspath(os.path.dirname(__file__)))
  sys.path.insert(0, os.path.join(PROJECT_DIR, "sfish"))
  
  RANDOM_UA_TYPE = "random"
  
  USER_AGENT = [
      'Mozilla/5.0 (Windows NT 6.1; rv:21.0) Gecko/20130401 Firefox/21.0Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.2117.157 Safari/537.36',
      'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.15 (KHTML, like Gecko) Chrome/24.0.1295.0 Safari/537.15',
      'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:18.0)  Gecko/20100101 Firefox/18.0',
      'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36',
      'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/29.0.1547.62 Safari/537.36',
      'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.3319.102 Safari/537.36',
      'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2227.0 Safari/537.36',
  
      'Mozilla/5.0 (Windows NT 6.1; WOW64; rv:21.0) Gecko/20130330 Firefox/21.0',
      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36',
      'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1467.0 Safari/537.36',
      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1664.3 Safari/537.36',
      'Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1468.0 Safari/537.36',
      'Mozilla/5.0 (Windows NT 6.2; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1500.55 Safari/537.36',
      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_8_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1664.3 Safari/537.36',
      'Mozilla/5.0 (Windows NT 6.2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1467.0 Safari/537.36',
  
      'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/27.0.1453.93 Safari/537.36',
      'Mozilla/5.0 (Windows NT 10.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/40.0.2214.93 Safari/537.36',
      'Mozilla/5.0 (Windows NT 6.1; rv:21.0) Gecko/20130401 Firefox/21.0',
  
  
      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1664.3 Safari/537.36',
      'Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2224.3 Safari/537.36',
  
      'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1664.3 Safari/537.36',
      'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.137 Safari/4E423F'
  
  ]
  