text: # -*- coding: utf-8 -*-
  
  # Scrapy settings for jobs project
  #
  # For simplicity, this file contains only settings considered important or
  # commonly used. You can find more settings consulting the documentation:
  #
  #     http://doc.scrapy.org/en/latest/topics/settings.html
  #     http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html
  #     http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html
  
  BOT_NAME = 'jobs'
  
  SPIDER_MODULES = ['jobs.spiders']
  NEWSPIDER_MODULE = 'jobs.spiders'
  
  # Crawl responsibly by identifying yourself (and your website) on the user-agent
  #USER_AGENT = 'jobs (+http://www.yourdomain.com)'
  
  # Obey robots.txt rules
  #ROBOTSTXT_OBEY = True
  
  # Configure maximum concurrent requests performed by Scrapy (default: 16)
  CONCURRENT_REQUESTS = 16
  
  # Configure a delay for requests for the same website (default: 0)
  # See http://scrapy.readthedocs.org/en/latest/topics/settings.html#download-delay
  # See also autothrottle settings and docs
  DOWNLOAD_DELAY = 1
  # The download delay setting will honor only one of:
  CONCURRENT_REQUESTS_PER_DOMAIN = 16
  CONCURRENT_REQUESTS_PER_IP = 16
  
  # Disable cookies (enabled by default)
  COOKIES_ENABLED = False
  
  # Disable Telnet Console (enabled by default)
  #TELNETCONSOLE_ENABLED = False
  
  # Override the default request headers:
  #DEFAULT_REQUEST_HEADERS = {
  #   'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
  #   'Accept-Language': 'en',
  #}
  
  # Enable or disable spider middlewares
  # See http://scrapy.readthedocs.org/en/latest/topics/spider-middleware.html
  #SPIDER_MIDDLEWARES = {
  #    'jobs.middlewares.JobsSpiderMiddleware': 543,
  #}
  
  # Enable or disable downloader middlewares
  # See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html
  DOWNLOADER_MIDDLEWARES = {
      # Disable inner user-agent middleware
      'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': None,
      'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware':
      350,
      # Enable custom user-agent middleware
      'jobs.middlewares.RandomUserAgentMiddleware': 500,
      'scrapy.downloadermiddlewares.retry.RetryMiddleware': 550,
      'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 750,
      # 'jobs.middlewares.RandomHttpProxyMiddleware': 999,
      # for lagou
      #'jobs.middlewares.RandomUserAgentMiddleware': 300,
      #'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 500,
  }
  
  # Enable or disable extensions
  # See http://scrapy.readthedocs.org/en/latest/topics/extensions.html
  #EXTENSIONS = {
  #    'scrapy.extensions.telnet.TelnetConsole': None,
  #}
  
  # Configure item pipelines
  # See http://scrapy.readthedocs.org/en/latest/topics/item-pipeline.html
  ITEM_PIPELINES = {
      'jobs.pipelines.DuplicatesPipeline': 300,
      'jobs.pipelines.MongoPipeline': 500,
  }
  
  # Enable and configure the AutoThrottle extension (disabled by default)
  # See http://doc.scrapy.org/en/latest/topics/autothrottle.html
  AUTOTHROTTLE_ENABLED = True
  # The initial download delay
  AUTOTHROTTLE_START_DELAY = 5
  # The maximum download delay to be set in case of high latencies
  AUTOTHROTTLE_MAX_DELAY = 60
  # The average number of requests Scrapy should be sending in parallel to
  # each remote server
  AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
  # Enable showing throttling stats for every response received:
  AUTOTHROTTLE_DEBUG = False
  
  # Enable and configure HTTP caching (disabled by default)
  # See http://scrapy.readthedocs.org/en/latest/topics/downloader-middleware.html#httpcache-middleware-settings
  #HTTPCACHE_ENABLED = True
  #HTTPCACHE_EXPIRATION_SECS = 0
  #HTTPCACHE_DIR = 'httpcache'
  #HTTPCACHE_IGNORE_HTTP_CODES = []
  #HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
  
  # Feed exports
  # FEED_FORMAT = 'csv'
  # from os import path
  # d = path.dirname(__file__)
  # FEED_URI = 'file://' + d + '/scraping/feeds/%(name)s/%(time)s.csv'
  
  # Custom configuration
  MONGO_URI = 'mongodb://localhost:27017/'
  MONGO_DATABASE = 'jobs'
  
  # Proxy config
  PROXY_MODE = 0
  PROXY_FILE = 'proxies.dat'
  
  # Retry many times since proxies often fail
  RETRY_TIMES = 5
  # Retry on most error codes since proxies fail for different reasons
  RETRY_HTTP_CODES = [500, 503, 504, 400, 403, 404, 408]
  