We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
DEFAULT_REQUEST_HEADERS can't set User-Agent
If I use DEFAULT_REQUEST_HEADERS to set `User-Agent`, it doesn't work.

``` python
DEFAULT_REQUEST_HEADERS = {
    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.84 Safari/537.36',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
    'Accept-Encoding': 'gzip, deflate, sdch',
    'Accept-Language': 'en-US,en;q=0.8,zh-CN;q=0.6,zh;q=0.4',
}
```

I know I can set `User-Agent` for  crawler  by using USER_AGENT setting key:

``` python
# settings.py
USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.84 Safari/537.36'
```

I think this behaviour is not corresponding with the doc [User Agent](http://doc.scrapy.org/en/latest/topics/settings.html#user-agent) and [DefaultHeadersMiddleware](http://doc.scrapy.org/en/latest/topics/downloader-middleware.html#scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware) (_This middleware sets all default requests headers specified in the DEFAULT_REQUEST_HEADERS setting._)

If this behaviour is designed, maybe the doc should be modified.

Thanks.


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `scrapy/settings/default_settings.py`
Content:
```
1 """
2 This module contains the default values for all settings used by Scrapy.
3 
4 For more information about these settings you can read the settings
5 documentation in docs/topics/settings.rst
6 
7 Scrapy developers, if you add a setting here remember to:
8 
9 * add it in alphabetical order
10 * group similar settings without leaving blank lines
11 * add its documentation to the available settings documentation
12   (docs/topics/settings.rst)
13 
14 """
15 
16 import os
17 import sys
18 from importlib import import_module
19 from os.path import join, abspath, dirname
20 
21 import six
22 
23 AJAXCRAWL_ENABLED = False
24 
25 AUTOTHROTTLE_ENABLED = False
26 AUTOTHROTTLE_DEBUG = False
27 AUTOTHROTTLE_MAX_DELAY = 60.0
28 AUTOTHROTTLE_START_DELAY = 5.0
29 AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
30 
31 BOT_NAME = 'scrapybot'
32 
33 CLOSESPIDER_TIMEOUT = 0
34 CLOSESPIDER_PAGECOUNT = 0
35 CLOSESPIDER_ITEMCOUNT = 0
36 CLOSESPIDER_ERRORCOUNT = 0
37 
38 COMMANDS_MODULE = ''
39 
40 COMPRESSION_ENABLED = True
41 
42 CONCURRENT_ITEMS = 100
43 
44 CONCURRENT_REQUESTS = 16
45 CONCURRENT_REQUESTS_PER_DOMAIN = 8
46 CONCURRENT_REQUESTS_PER_IP = 0
47 
48 COOKIES_ENABLED = True
49 COOKIES_DEBUG = False
50 
51 DEFAULT_ITEM_CLASS = 'scrapy.item.Item'
52 
53 DEFAULT_REQUEST_HEADERS = {
54     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
55     'Accept-Language': 'en',
56 }
57 
58 DEPTH_LIMIT = 0
59 DEPTH_STATS = True
60 DEPTH_PRIORITY = 0
61 
62 DNSCACHE_ENABLED = True
63 DNSCACHE_SIZE = 10000
64 DNS_TIMEOUT = 60
65 
66 DOWNLOAD_DELAY = 0
67 
68 DOWNLOAD_HANDLERS = {}
69 DOWNLOAD_HANDLERS_BASE = {
70     'file': 'scrapy.core.downloader.handlers.file.FileDownloadHandler',
71     'http': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',
72     'https': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',
73     's3': 'scrapy.core.downloader.handlers.s3.S3DownloadHandler',
74     'ftp': 'scrapy.core.downloader.handlers.ftp.FTPDownloadHandler',
75 }
76 
77 DOWNLOAD_TIMEOUT = 180      # 3mins
78 
79 DOWNLOAD_MAXSIZE = 1024*1024*1024   # 1024m
80 DOWNLOAD_WARNSIZE = 32*1024*1024    # 32m
81 
82 DOWNLOADER = 'scrapy.core.downloader.Downloader'
83 
84 DOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory'
85 DOWNLOADER_CLIENTCONTEXTFACTORY = 'scrapy.core.downloader.contextfactory.ScrapyClientContextFactory'
86 DOWNLOADER_CLIENT_TLS_METHOD = 'TLS' # Use highest TLS/SSL protocol version supported by the platform,
87                                      # also allowing negotiation
88 
89 DOWNLOADER_MIDDLEWARES = {}
90 
91 DOWNLOADER_MIDDLEWARES_BASE = {
92     # Engine side
93     'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100,
94     'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300,
95     'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350,
96     'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 400,
97     'scrapy.downloadermiddlewares.retry.RetryMiddleware': 500,
98     'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 550,
99     'scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware': 560,
100     'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware': 580,
101     'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 590,
102     'scrapy.downloadermiddlewares.redirect.RedirectMiddleware': 600,
103     'scrapy.downloadermiddlewares.cookies.CookiesMiddleware': 700,
104     'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 750,
105     'scrapy.downloadermiddlewares.chunked.ChunkedTransferMiddleware': 830,
106     'scrapy.downloadermiddlewares.stats.DownloaderStats': 850,
107     'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware': 900,
108     # Downloader side
109 }
110 
111 DOWNLOADER_STATS = True
112 
113 DUPEFILTER_CLASS = 'scrapy.dupefilters.RFPDupeFilter'
114 
115 try:
116     EDITOR = os.environ['EDITOR']
117 except KeyError:
118     if sys.platform == 'win32':
119         EDITOR = '%s -m idlelib.idle'
120     else:
121         EDITOR = 'vi'
122 
123 EXTENSIONS = {}
124 
125 EXTENSIONS_BASE = {
126     'scrapy.extensions.corestats.CoreStats': 0,
127     'scrapy.extensions.telnet.TelnetConsole': 0,
128     'scrapy.extensions.memusage.MemoryUsage': 0,
129     'scrapy.extensions.memdebug.MemoryDebugger': 0,
130     'scrapy.extensions.closespider.CloseSpider': 0,
131     'scrapy.extensions.feedexport.FeedExporter': 0,
132     'scrapy.extensions.logstats.LogStats': 0,
133     'scrapy.extensions.spiderstate.SpiderState': 0,
134     'scrapy.extensions.throttle.AutoThrottle': 0,
135 }
136 
137 FEED_TEMPDIR = None
138 FEED_URI = None
139 FEED_URI_PARAMS = None  # a function to extend uri arguments
140 FEED_FORMAT = 'jsonlines'
141 FEED_STORE_EMPTY = False
142 FEED_EXPORT_FIELDS = None
143 FEED_STORAGES = {}
144 FEED_STORAGES_BASE = {
145     '': 'scrapy.extensions.feedexport.FileFeedStorage',
146     'file': 'scrapy.extensions.feedexport.FileFeedStorage',
147     'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage',
148     's3': 'scrapy.extensions.feedexport.S3FeedStorage',
149     'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
150 }
151 FEED_EXPORTERS = {}
152 FEED_EXPORTERS_BASE = {
153     'json': 'scrapy.exporters.JsonItemExporter',
154     'jsonlines': 'scrapy.exporters.JsonLinesItemExporter',
155     'jl': 'scrapy.exporters.JsonLinesItemExporter',
156     'csv': 'scrapy.exporters.CsvItemExporter',
157     'xml': 'scrapy.exporters.XmlItemExporter',
158     'marshal': 'scrapy.exporters.MarshalItemExporter',
159     'pickle': 'scrapy.exporters.PickleItemExporter',
160 }
161 
162 FILES_STORE_S3_ACL = 'private'
163 FILES_EXPIRES = 90
164 FILES_URLS_FIELD = 'file_urls'
165 FILES_RESULT_FIELD = 'files'
166 
167 HTTPCACHE_ENABLED = False
168 HTTPCACHE_DIR = 'httpcache'
169 HTTPCACHE_IGNORE_MISSING = False
170 HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
171 HTTPCACHE_EXPIRATION_SECS = 0
172 HTTPCACHE_ALWAYS_STORE = False
173 HTTPCACHE_IGNORE_HTTP_CODES = []
174 HTTPCACHE_IGNORE_SCHEMES = ['file']
175 HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS = []
176 HTTPCACHE_DBM_MODULE = 'anydbm' if six.PY2 else 'dbm'
177 HTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy'
178 HTTPCACHE_GZIP = False
179 
180 HTTPPROXY_AUTH_ENCODING = 'latin-1'
181 
182 IMAGES_MIN_WIDTH = 0
183 IMAGES_MIN_HEIGHT = 0
184 IMAGES_EXPIRES = 90
185 IMAGES_THUMBS = {}
186 IMAGES_URLS_FIELD = 'image_urls'
187 IMAGES_RESULT_FIELD = 'images'
188 
189 ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager'
190 
191 ITEM_PIPELINES = {}
192 ITEM_PIPELINES_BASE = {}
193 
194 LOG_ENABLED = True
195 LOG_ENCODING = 'utf-8'
196 LOG_FORMATTER = 'scrapy.logformatter.LogFormatter'
197 LOG_FORMAT = '%(asctime)s [%(name)s] %(levelname)s: %(message)s'
198 LOG_DATEFORMAT = '%Y-%m-%d %H:%M:%S'
199 LOG_STDOUT = False
200 LOG_LEVEL = 'DEBUG'
201 LOG_FILE = None
202 
203 LOG_UNSERIALIZABLE_REQUESTS = False
204 
205 LOGSTATS_INTERVAL = 60.0
206 
207 MAIL_HOST = 'localhost'
208 MAIL_PORT = 25
209 MAIL_FROM = 'scrapy@localhost'
210 MAIL_PASS = None
211 MAIL_USER = None
212 
213 MEMDEBUG_ENABLED = False        # enable memory debugging
214 MEMDEBUG_NOTIFY = []            # send memory debugging report by mail at engine shutdown
215 
216 MEMUSAGE_CHECK_INTERVAL_SECONDS = 60.0
217 MEMUSAGE_ENABLED = False
218 MEMUSAGE_LIMIT_MB = 0
219 MEMUSAGE_NOTIFY_MAIL = []
220 MEMUSAGE_REPORT = False
221 MEMUSAGE_WARNING_MB = 0
222 
223 METAREFRESH_ENABLED = True
224 METAREFRESH_MAXDELAY = 100
225 
226 NEWSPIDER_MODULE = ''
227 
228 RANDOMIZE_DOWNLOAD_DELAY = True
229 
230 REACTOR_THREADPOOL_MAXSIZE = 10
231 
232 REDIRECT_ENABLED = True
233 REDIRECT_MAX_TIMES = 20  # uses Firefox default setting
234 REDIRECT_PRIORITY_ADJUST = +2
235 
236 REFERER_ENABLED = True
237 
238 RETRY_ENABLED = True
239 RETRY_TIMES = 2  # initial response + 2 retries = 3 requests
240 RETRY_HTTP_CODES = [500, 502, 503, 504, 408]
241 RETRY_PRIORITY_ADJUST = -1
242 
243 ROBOTSTXT_OBEY = False
244 
245 SCHEDULER = 'scrapy.core.scheduler.Scheduler'
246 SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleLifoDiskQueue'
247 SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.LifoMemoryQueue'
248 SCHEDULER_PRIORITY_QUEUE = 'queuelib.PriorityQueue'
249 
250 SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader'
251 
252 SPIDER_MIDDLEWARES = {}
253 
254 SPIDER_MIDDLEWARES_BASE = {
255     # Engine side
256     'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware': 50,
257     'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': 500,
258     'scrapy.spidermiddlewares.referer.RefererMiddleware': 700,
259     'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware': 800,
260     'scrapy.spidermiddlewares.depth.DepthMiddleware': 900,
261     # Spider side
262 }
263 
264 SPIDER_MODULES = []
265 
266 STATS_CLASS = 'scrapy.statscollectors.MemoryStatsCollector'
267 STATS_DUMP = True
268 
269 STATSMAILER_RCPTS = []
270 
271 TEMPLATES_DIR = abspath(join(dirname(__file__), '..', 'templates'))
272 
273 URLLENGTH_LIMIT = 2083
274 
275 USER_AGENT = 'Scrapy/%s (+http://scrapy.org)' % import_module('scrapy').__version__
276 
277 TELNETCONSOLE_ENABLED = 1
278 TELNETCONSOLE_PORT = [6023, 6073]
279 TELNETCONSOLE_HOST = '127.0.0.1'
280 
281 SPIDER_CONTRACTS = {}
282 SPIDER_CONTRACTS_BASE = {
283     'scrapy.contracts.default.UrlContract': 1,
284     'scrapy.contracts.default.ReturnsContract': 2,
285     'scrapy.contracts.default.ScrapesContract': 3,
286 }
287
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -93,9 +93,9 @@
     'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100,
     'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300,
     'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350,
-    'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 400,
-    'scrapy.downloadermiddlewares.retry.RetryMiddleware': 500,
-    'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 550,
+    'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 400,
+    'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 500,
+    'scrapy.downloadermiddlewares.retry.RetryMiddleware': 550,
     'scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware': 560,
     'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware': 580,
     'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 590,

{"golden_diff": "diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py\n--- a/scrapy/settings/default_settings.py\n+++ b/scrapy/settings/default_settings.py\n@@ -93,9 +93,9 @@\n     'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100,\n     'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300,\n     'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350,\n-    'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 400,\n-    'scrapy.downloadermiddlewares.retry.RetryMiddleware': 500,\n-    'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 550,\n+    'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 400,\n+    'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 500,\n+    'scrapy.downloadermiddlewares.retry.RetryMiddleware': 550,\n     'scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware': 560,\n     'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware': 580,\n     'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 590,\n", "issue": "DEFAULT_REQUEST_HEADERS can't set User-Agent\nIf I use DEFAULT_REQUEST_HEADERS to set `User-Agent`, it doesn't work.\n\n``` python\nDEFAULT_REQUEST_HEADERS = {\n    'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.84 Safari/537.36',\n    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',\n    'Accept-Encoding': 'gzip, deflate, sdch',\n    'Accept-Language': 'en-US,en;q=0.8,zh-CN;q=0.6,zh;q=0.4',\n}\n```\n\nI know I can set `User-Agent` for  crawler  by using USER_AGENT setting key:\n\n``` python\n# settings.py\nUSER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.84 Safari/537.36'\n```\n\nI think this behaviour is not corresponding with the doc [User Agent](http://doc.scrapy.org/en/latest/topics/settings.html#user-agent) and [DefaultHeadersMiddleware](http://doc.scrapy.org/en/latest/topics/downloader-middleware.html#scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware) (_This middleware sets all default requests headers specified in the DEFAULT_REQUEST_HEADERS setting._)\n\nIf this behaviour is designed, maybe the doc should be modified.\n\nThanks.\n\n", "before_files": [{"content": "\"\"\"\nThis module contains the default values for all settings used by Scrapy.\n\nFor more information about these settings you can read the settings\ndocumentation in docs/topics/settings.rst\n\nScrapy developers, if you add a setting here remember to:\n\n* add it in alphabetical order\n* group similar settings without leaving blank lines\n* add its documentation to the available settings documentation\n  (docs/topics/settings.rst)\n\n\"\"\"\n\nimport os\nimport sys\nfrom importlib import import_module\nfrom os.path import join, abspath, dirname\n\nimport six\n\nAJAXCRAWL_ENABLED = False\n\nAUTOTHROTTLE_ENABLED = False\nAUTOTHROTTLE_DEBUG = False\nAUTOTHROTTLE_MAX_DELAY = 60.0\nAUTOTHROTTLE_START_DELAY = 5.0\nAUTOTHROTTLE_TARGET_CONCURRENCY = 1.0\n\nBOT_NAME = 'scrapybot'\n\nCLOSESPIDER_TIMEOUT = 0\nCLOSESPIDER_PAGECOUNT = 0\nCLOSESPIDER_ITEMCOUNT = 0\nCLOSESPIDER_ERRORCOUNT = 0\n\nCOMMANDS_MODULE = ''\n\nCOMPRESSION_ENABLED = True\n\nCONCURRENT_ITEMS = 100\n\nCONCURRENT_REQUESTS = 16\nCONCURRENT_REQUESTS_PER_DOMAIN = 8\nCONCURRENT_REQUESTS_PER_IP = 0\n\nCOOKIES_ENABLED = True\nCOOKIES_DEBUG = False\n\nDEFAULT_ITEM_CLASS = 'scrapy.item.Item'\n\nDEFAULT_REQUEST_HEADERS = {\n    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',\n    'Accept-Language': 'en',\n}\n\nDEPTH_LIMIT = 0\nDEPTH_STATS = True\nDEPTH_PRIORITY = 0\n\nDNSCACHE_ENABLED = True\nDNSCACHE_SIZE = 10000\nDNS_TIMEOUT = 60\n\nDOWNLOAD_DELAY = 0\n\nDOWNLOAD_HANDLERS = {}\nDOWNLOAD_HANDLERS_BASE = {\n    'file': 'scrapy.core.downloader.handlers.file.FileDownloadHandler',\n    'http': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',\n    'https': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',\n    's3': 'scrapy.core.downloader.handlers.s3.S3DownloadHandler',\n    'ftp': 'scrapy.core.downloader.handlers.ftp.FTPDownloadHandler',\n}\n\nDOWNLOAD_TIMEOUT = 180      # 3mins\n\nDOWNLOAD_MAXSIZE = 1024*1024*1024   # 1024m\nDOWNLOAD_WARNSIZE = 32*1024*1024    # 32m\n\nDOWNLOADER = 'scrapy.core.downloader.Downloader'\n\nDOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory'\nDOWNLOADER_CLIENTCONTEXTFACTORY = 'scrapy.core.downloader.contextfactory.ScrapyClientContextFactory'\nDOWNLOADER_CLIENT_TLS_METHOD = 'TLS' # Use highest TLS/SSL protocol version supported by the platform,\n                                     # also allowing negotiation\n\nDOWNLOADER_MIDDLEWARES = {}\n\nDOWNLOADER_MIDDLEWARES_BASE = {\n    # Engine side\n    'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100,\n    'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300,\n    'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350,\n    'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 400,\n    'scrapy.downloadermiddlewares.retry.RetryMiddleware': 500,\n    'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 550,\n    'scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware': 560,\n    'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware': 580,\n    'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 590,\n    'scrapy.downloadermiddlewares.redirect.RedirectMiddleware': 600,\n    'scrapy.downloadermiddlewares.cookies.CookiesMiddleware': 700,\n    'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 750,\n    'scrapy.downloadermiddlewares.chunked.ChunkedTransferMiddleware': 830,\n    'scrapy.downloadermiddlewares.stats.DownloaderStats': 850,\n    'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware': 900,\n    # Downloader side\n}\n\nDOWNLOADER_STATS = True\n\nDUPEFILTER_CLASS = 'scrapy.dupefilters.RFPDupeFilter'\n\ntry:\n    EDITOR = os.environ['EDITOR']\nexcept KeyError:\n    if sys.platform == 'win32':\n        EDITOR = '%s -m idlelib.idle'\n    else:\n        EDITOR = 'vi'\n\nEXTENSIONS = {}\n\nEXTENSIONS_BASE = {\n    'scrapy.extensions.corestats.CoreStats': 0,\n    'scrapy.extensions.telnet.TelnetConsole': 0,\n    'scrapy.extensions.memusage.MemoryUsage': 0,\n    'scrapy.extensions.memdebug.MemoryDebugger': 0,\n    'scrapy.extensions.closespider.CloseSpider': 0,\n    'scrapy.extensions.feedexport.FeedExporter': 0,\n    'scrapy.extensions.logstats.LogStats': 0,\n    'scrapy.extensions.spiderstate.SpiderState': 0,\n    'scrapy.extensions.throttle.AutoThrottle': 0,\n}\n\nFEED_TEMPDIR = None\nFEED_URI = None\nFEED_URI_PARAMS = None  # a function to extend uri arguments\nFEED_FORMAT = 'jsonlines'\nFEED_STORE_EMPTY = False\nFEED_EXPORT_FIELDS = None\nFEED_STORAGES = {}\nFEED_STORAGES_BASE = {\n    '': 'scrapy.extensions.feedexport.FileFeedStorage',\n    'file': 'scrapy.extensions.feedexport.FileFeedStorage',\n    'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage',\n    's3': 'scrapy.extensions.feedexport.S3FeedStorage',\n    'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',\n}\nFEED_EXPORTERS = {}\nFEED_EXPORTERS_BASE = {\n    'json': 'scrapy.exporters.JsonItemExporter',\n    'jsonlines': 'scrapy.exporters.JsonLinesItemExporter',\n    'jl': 'scrapy.exporters.JsonLinesItemExporter',\n    'csv': 'scrapy.exporters.CsvItemExporter',\n    'xml': 'scrapy.exporters.XmlItemExporter',\n    'marshal': 'scrapy.exporters.MarshalItemExporter',\n    'pickle': 'scrapy.exporters.PickleItemExporter',\n}\n\nFILES_STORE_S3_ACL = 'private'\nFILES_EXPIRES = 90\nFILES_URLS_FIELD = 'file_urls'\nFILES_RESULT_FIELD = 'files'\n\nHTTPCACHE_ENABLED = False\nHTTPCACHE_DIR = 'httpcache'\nHTTPCACHE_IGNORE_MISSING = False\nHTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'\nHTTPCACHE_EXPIRATION_SECS = 0\nHTTPCACHE_ALWAYS_STORE = False\nHTTPCACHE_IGNORE_HTTP_CODES = []\nHTTPCACHE_IGNORE_SCHEMES = ['file']\nHTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS = []\nHTTPCACHE_DBM_MODULE = 'anydbm' if six.PY2 else 'dbm'\nHTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy'\nHTTPCACHE_GZIP = False\n\nHTTPPROXY_AUTH_ENCODING = 'latin-1'\n\nIMAGES_MIN_WIDTH = 0\nIMAGES_MIN_HEIGHT = 0\nIMAGES_EXPIRES = 90\nIMAGES_THUMBS = {}\nIMAGES_URLS_FIELD = 'image_urls'\nIMAGES_RESULT_FIELD = 'images'\n\nITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager'\n\nITEM_PIPELINES = {}\nITEM_PIPELINES_BASE = {}\n\nLOG_ENABLED = True\nLOG_ENCODING = 'utf-8'\nLOG_FORMATTER = 'scrapy.logformatter.LogFormatter'\nLOG_FORMAT = '%(asctime)s [%(name)s] %(levelname)s: %(message)s'\nLOG_DATEFORMAT = '%Y-%m-%d %H:%M:%S'\nLOG_STDOUT = False\nLOG_LEVEL = 'DEBUG'\nLOG_FILE = None\n\nLOG_UNSERIALIZABLE_REQUESTS = False\n\nLOGSTATS_INTERVAL = 60.0\n\nMAIL_HOST = 'localhost'\nMAIL_PORT = 25\nMAIL_FROM = 'scrapy@localhost'\nMAIL_PASS = None\nMAIL_USER = None\n\nMEMDEBUG_ENABLED = False        # enable memory debugging\nMEMDEBUG_NOTIFY = []            # send memory debugging report by mail at engine shutdown\n\nMEMUSAGE_CHECK_INTERVAL_SECONDS = 60.0\nMEMUSAGE_ENABLED = False\nMEMUSAGE_LIMIT_MB = 0\nMEMUSAGE_NOTIFY_MAIL = []\nMEMUSAGE_REPORT = False\nMEMUSAGE_WARNING_MB = 0\n\nMETAREFRESH_ENABLED = True\nMETAREFRESH_MAXDELAY = 100\n\nNEWSPIDER_MODULE = ''\n\nRANDOMIZE_DOWNLOAD_DELAY = True\n\nREACTOR_THREADPOOL_MAXSIZE = 10\n\nREDIRECT_ENABLED = True\nREDIRECT_MAX_TIMES = 20  # uses Firefox default setting\nREDIRECT_PRIORITY_ADJUST = +2\n\nREFERER_ENABLED = True\n\nRETRY_ENABLED = True\nRETRY_TIMES = 2  # initial response + 2 retries = 3 requests\nRETRY_HTTP_CODES = [500, 502, 503, 504, 408]\nRETRY_PRIORITY_ADJUST = -1\n\nROBOTSTXT_OBEY = False\n\nSCHEDULER = 'scrapy.core.scheduler.Scheduler'\nSCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleLifoDiskQueue'\nSCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.LifoMemoryQueue'\nSCHEDULER_PRIORITY_QUEUE = 'queuelib.PriorityQueue'\n\nSPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader'\n\nSPIDER_MIDDLEWARES = {}\n\nSPIDER_MIDDLEWARES_BASE = {\n    # Engine side\n    'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware': 50,\n    'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': 500,\n    'scrapy.spidermiddlewares.referer.RefererMiddleware': 700,\n    'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware': 800,\n    'scrapy.spidermiddlewares.depth.DepthMiddleware': 900,\n    # Spider side\n}\n\nSPIDER_MODULES = []\n\nSTATS_CLASS = 'scrapy.statscollectors.MemoryStatsCollector'\nSTATS_DUMP = True\n\nSTATSMAILER_RCPTS = []\n\nTEMPLATES_DIR = abspath(join(dirname(__file__), '..', 'templates'))\n\nURLLENGTH_LIMIT = 2083\n\nUSER_AGENT = 'Scrapy/%s (+http://scrapy.org)' % import_module('scrapy').__version__\n\nTELNETCONSOLE_ENABLED = 1\nTELNETCONSOLE_PORT = [6023, 6073]\nTELNETCONSOLE_HOST = '127.0.0.1'\n\nSPIDER_CONTRACTS = {}\nSPIDER_CONTRACTS_BASE = {\n    'scrapy.contracts.default.UrlContract': 1,\n    'scrapy.contracts.default.ReturnsContract': 2,\n    'scrapy.contracts.default.ScrapesContract': 3,\n}\n", "path": "scrapy/settings/default_settings.py"}], "after_files": [{"content": "\"\"\"\nThis module contains the default values for all settings used by Scrapy.\n\nFor more information about these settings you can read the settings\ndocumentation in docs/topics/settings.rst\n\nScrapy developers, if you add a setting here remember to:\n\n* add it in alphabetical order\n* group similar settings without leaving blank lines\n* add its documentation to the available settings documentation\n  (docs/topics/settings.rst)\n\n\"\"\"\n\nimport os\nimport sys\nfrom importlib import import_module\nfrom os.path import join, abspath, dirname\n\nimport six\n\nAJAXCRAWL_ENABLED = False\n\nAUTOTHROTTLE_ENABLED = False\nAUTOTHROTTLE_DEBUG = False\nAUTOTHROTTLE_MAX_DELAY = 60.0\nAUTOTHROTTLE_START_DELAY = 5.0\nAUTOTHROTTLE_TARGET_CONCURRENCY = 1.0\n\nBOT_NAME = 'scrapybot'\n\nCLOSESPIDER_TIMEOUT = 0\nCLOSESPIDER_PAGECOUNT = 0\nCLOSESPIDER_ITEMCOUNT = 0\nCLOSESPIDER_ERRORCOUNT = 0\n\nCOMMANDS_MODULE = ''\n\nCOMPRESSION_ENABLED = True\n\nCONCURRENT_ITEMS = 100\n\nCONCURRENT_REQUESTS = 16\nCONCURRENT_REQUESTS_PER_DOMAIN = 8\nCONCURRENT_REQUESTS_PER_IP = 0\n\nCOOKIES_ENABLED = True\nCOOKIES_DEBUG = False\n\nDEFAULT_ITEM_CLASS = 'scrapy.item.Item'\n\nDEFAULT_REQUEST_HEADERS = {\n    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',\n    'Accept-Language': 'en',\n}\n\nDEPTH_LIMIT = 0\nDEPTH_STATS = True\nDEPTH_PRIORITY = 0\n\nDNSCACHE_ENABLED = True\nDNSCACHE_SIZE = 10000\nDNS_TIMEOUT = 60\n\nDOWNLOAD_DELAY = 0\n\nDOWNLOAD_HANDLERS = {}\nDOWNLOAD_HANDLERS_BASE = {\n    'file': 'scrapy.core.downloader.handlers.file.FileDownloadHandler',\n    'http': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',\n    'https': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',\n    's3': 'scrapy.core.downloader.handlers.s3.S3DownloadHandler',\n    'ftp': 'scrapy.core.downloader.handlers.ftp.FTPDownloadHandler',\n}\n\nDOWNLOAD_TIMEOUT = 180      # 3mins\n\nDOWNLOAD_MAXSIZE = 1024*1024*1024   # 1024m\nDOWNLOAD_WARNSIZE = 32*1024*1024    # 32m\n\nDOWNLOADER = 'scrapy.core.downloader.Downloader'\n\nDOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory'\nDOWNLOADER_CLIENTCONTEXTFACTORY = 'scrapy.core.downloader.contextfactory.ScrapyClientContextFactory'\nDOWNLOADER_CLIENT_TLS_METHOD = 'TLS' # Use highest TLS/SSL protocol version supported by the platform,\n                                     # also allowing negotiation\n\nDOWNLOADER_MIDDLEWARES = {}\n\nDOWNLOADER_MIDDLEWARES_BASE = {\n    # Engine side\n    'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100,\n    'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300,\n    'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350,\n    'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 400,\n    'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 500,\n    'scrapy.downloadermiddlewares.retry.RetryMiddleware': 550,\n    'scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware': 560,\n    'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware': 580,\n    'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 590,\n    'scrapy.downloadermiddlewares.redirect.RedirectMiddleware': 600,\n    'scrapy.downloadermiddlewares.cookies.CookiesMiddleware': 700,\n    'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 750,\n    'scrapy.downloadermiddlewares.chunked.ChunkedTransferMiddleware': 830,\n    'scrapy.downloadermiddlewares.stats.DownloaderStats': 850,\n    'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware': 900,\n    # Downloader side\n}\n\nDOWNLOADER_STATS = True\n\nDUPEFILTER_CLASS = 'scrapy.dupefilters.RFPDupeFilter'\n\ntry:\n    EDITOR = os.environ['EDITOR']\nexcept KeyError:\n    if sys.platform == 'win32':\n        EDITOR = '%s -m idlelib.idle'\n    else:\n        EDITOR = 'vi'\n\nEXTENSIONS = {}\n\nEXTENSIONS_BASE = {\n    'scrapy.extensions.corestats.CoreStats': 0,\n    'scrapy.extensions.telnet.TelnetConsole': 0,\n    'scrapy.extensions.memusage.MemoryUsage': 0,\n    'scrapy.extensions.memdebug.MemoryDebugger': 0,\n    'scrapy.extensions.closespider.CloseSpider': 0,\n    'scrapy.extensions.feedexport.FeedExporter': 0,\n    'scrapy.extensions.logstats.LogStats': 0,\n    'scrapy.extensions.spiderstate.SpiderState': 0,\n    'scrapy.extensions.throttle.AutoThrottle': 0,\n}\n\nFEED_TEMPDIR = None\nFEED_URI = None\nFEED_URI_PARAMS = None  # a function to extend uri arguments\nFEED_FORMAT = 'jsonlines'\nFEED_STORE_EMPTY = False\nFEED_EXPORT_FIELDS = None\nFEED_STORAGES = {}\nFEED_STORAGES_BASE = {\n    '': 'scrapy.extensions.feedexport.FileFeedStorage',\n    'file': 'scrapy.extensions.feedexport.FileFeedStorage',\n    'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage',\n    's3': 'scrapy.extensions.feedexport.S3FeedStorage',\n    'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',\n}\nFEED_EXPORTERS = {}\nFEED_EXPORTERS_BASE = {\n    'json': 'scrapy.exporters.JsonItemExporter',\n    'jsonlines': 'scrapy.exporters.JsonLinesItemExporter',\n    'jl': 'scrapy.exporters.JsonLinesItemExporter',\n    'csv': 'scrapy.exporters.CsvItemExporter',\n    'xml': 'scrapy.exporters.XmlItemExporter',\n    'marshal': 'scrapy.exporters.MarshalItemExporter',\n    'pickle': 'scrapy.exporters.PickleItemExporter',\n}\n\nFILES_STORE_S3_ACL = 'private'\nFILES_EXPIRES = 90\nFILES_URLS_FIELD = 'file_urls'\nFILES_RESULT_FIELD = 'files'\n\nHTTPCACHE_ENABLED = False\nHTTPCACHE_DIR = 'httpcache'\nHTTPCACHE_IGNORE_MISSING = False\nHTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'\nHTTPCACHE_EXPIRATION_SECS = 0\nHTTPCACHE_ALWAYS_STORE = False\nHTTPCACHE_IGNORE_HTTP_CODES = []\nHTTPCACHE_IGNORE_SCHEMES = ['file']\nHTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS = []\nHTTPCACHE_DBM_MODULE = 'anydbm' if six.PY2 else 'dbm'\nHTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy'\nHTTPCACHE_GZIP = False\n\nHTTPPROXY_AUTH_ENCODING = 'latin-1'\n\nIMAGES_MIN_WIDTH = 0\nIMAGES_MIN_HEIGHT = 0\nIMAGES_EXPIRES = 90\nIMAGES_THUMBS = {}\nIMAGES_URLS_FIELD = 'image_urls'\nIMAGES_RESULT_FIELD = 'images'\n\nITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager'\n\nITEM_PIPELINES = {}\nITEM_PIPELINES_BASE = {}\n\nLOG_ENABLED = True\nLOG_ENCODING = 'utf-8'\nLOG_FORMATTER = 'scrapy.logformatter.LogFormatter'\nLOG_FORMAT = '%(asctime)s [%(name)s] %(levelname)s: %(message)s'\nLOG_DATEFORMAT = '%Y-%m-%d %H:%M:%S'\nLOG_STDOUT = False\nLOG_LEVEL = 'DEBUG'\nLOG_FILE = None\n\nLOG_UNSERIALIZABLE_REQUESTS = False\n\nLOGSTATS_INTERVAL = 60.0\n\nMAIL_HOST = 'localhost'\nMAIL_PORT = 25\nMAIL_FROM = 'scrapy@localhost'\nMAIL_PASS = None\nMAIL_USER = None\n\nMEMDEBUG_ENABLED = False        # enable memory debugging\nMEMDEBUG_NOTIFY = []            # send memory debugging report by mail at engine shutdown\n\nMEMUSAGE_CHECK_INTERVAL_SECONDS = 60.0\nMEMUSAGE_ENABLED = False\nMEMUSAGE_LIMIT_MB = 0\nMEMUSAGE_NOTIFY_MAIL = []\nMEMUSAGE_REPORT = False\nMEMUSAGE_WARNING_MB = 0\n\nMETAREFRESH_ENABLED = True\nMETAREFRESH_MAXDELAY = 100\n\nNEWSPIDER_MODULE = ''\n\nRANDOMIZE_DOWNLOAD_DELAY = True\n\nREACTOR_THREADPOOL_MAXSIZE = 10\n\nREDIRECT_ENABLED = True\nREDIRECT_MAX_TIMES = 20  # uses Firefox default setting\nREDIRECT_PRIORITY_ADJUST = +2\n\nREFERER_ENABLED = True\n\nRETRY_ENABLED = True\nRETRY_TIMES = 2  # initial response + 2 retries = 3 requests\nRETRY_HTTP_CODES = [500, 502, 503, 504, 408]\nRETRY_PRIORITY_ADJUST = -1\n\nROBOTSTXT_OBEY = False\n\nSCHEDULER = 'scrapy.core.scheduler.Scheduler'\nSCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleLifoDiskQueue'\nSCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.LifoMemoryQueue'\nSCHEDULER_PRIORITY_QUEUE = 'queuelib.PriorityQueue'\n\nSPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader'\n\nSPIDER_MIDDLEWARES = {}\n\nSPIDER_MIDDLEWARES_BASE = {\n    # Engine side\n    'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware': 50,\n    'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': 500,\n    'scrapy.spidermiddlewares.referer.RefererMiddleware': 700,\n    'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware': 800,\n    'scrapy.spidermiddlewares.depth.DepthMiddleware': 900,\n    # Spider side\n}\n\nSPIDER_MODULES = []\n\nSTATS_CLASS = 'scrapy.statscollectors.MemoryStatsCollector'\nSTATS_DUMP = True\n\nSTATSMAILER_RCPTS = []\n\nTEMPLATES_DIR = abspath(join(dirname(__file__), '..', 'templates'))\n\nURLLENGTH_LIMIT = 2083\n\nUSER_AGENT = 'Scrapy/%s (+http://scrapy.org)' % import_module('scrapy').__version__\n\nTELNETCONSOLE_ENABLED = 1\nTELNETCONSOLE_PORT = [6023, 6073]\nTELNETCONSOLE_HOST = '127.0.0.1'\n\nSPIDER_CONTRACTS = {}\nSPIDER_CONTRACTS_BASE = {\n    'scrapy.contracts.default.UrlContract': 1,\n    'scrapy.contracts.default.ReturnsContract': 2,\n    'scrapy.contracts.default.ScrapesContract': 3,\n}\n", "path": "scrapy/settings/default_settings.py"}]}