We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Scrapy ignores proxy credentials when using "proxy" meta key
Code `yield Request(link, meta={'proxy': 'http://user:password@ip:port’})` ignores user:password. 
Problem is solved by using header "Proxy-Authorization" with base64, but it is better to implement it inside Scrapy.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `scrapy/settings/default_settings.py`
Content:
```
1 """
2 This module contains the default values for all settings used by Scrapy.
3 
4 For more information about these settings you can read the settings
5 documentation in docs/topics/settings.rst
6 
7 Scrapy developers, if you add a setting here remember to:
8 
9 * add it in alphabetical order
10 * group similar settings without leaving blank lines
11 * add its documentation to the available settings documentation
12   (docs/topics/settings.rst)
13 
14 """
15 
16 import os
17 import sys
18 from importlib import import_module
19 from os.path import join, abspath, dirname
20 
21 import six
22 
23 AJAXCRAWL_ENABLED = False
24 
25 AUTOTHROTTLE_ENABLED = False
26 AUTOTHROTTLE_DEBUG = False
27 AUTOTHROTTLE_MAX_DELAY = 60.0
28 AUTOTHROTTLE_START_DELAY = 5.0
29 AUTOTHROTTLE_TARGET_CONCURRENCY = 1.0
30 
31 BOT_NAME = 'scrapybot'
32 
33 CLOSESPIDER_TIMEOUT = 0
34 CLOSESPIDER_PAGECOUNT = 0
35 CLOSESPIDER_ITEMCOUNT = 0
36 CLOSESPIDER_ERRORCOUNT = 0
37 
38 COMMANDS_MODULE = ''
39 
40 COMPRESSION_ENABLED = True
41 
42 CONCURRENT_ITEMS = 100
43 
44 CONCURRENT_REQUESTS = 16
45 CONCURRENT_REQUESTS_PER_DOMAIN = 8
46 CONCURRENT_REQUESTS_PER_IP = 0
47 
48 COOKIES_ENABLED = True
49 COOKIES_DEBUG = False
50 
51 DEFAULT_ITEM_CLASS = 'scrapy.item.Item'
52 
53 DEFAULT_REQUEST_HEADERS = {
54     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
55     'Accept-Language': 'en',
56 }
57 
58 DEPTH_LIMIT = 0
59 DEPTH_STATS = True
60 DEPTH_PRIORITY = 0
61 
62 DNSCACHE_ENABLED = True
63 DNSCACHE_SIZE = 10000
64 DNS_TIMEOUT = 60
65 
66 DOWNLOAD_DELAY = 0
67 
68 DOWNLOAD_HANDLERS = {}
69 DOWNLOAD_HANDLERS_BASE = {
70     'file': 'scrapy.core.downloader.handlers.file.FileDownloadHandler',
71     'http': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',
72     'https': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',
73     's3': 'scrapy.core.downloader.handlers.s3.S3DownloadHandler',
74     'ftp': 'scrapy.core.downloader.handlers.ftp.FTPDownloadHandler',
75 }
76 
77 DOWNLOAD_TIMEOUT = 180      # 3mins
78 
79 DOWNLOAD_MAXSIZE = 1024*1024*1024   # 1024m
80 DOWNLOAD_WARNSIZE = 32*1024*1024    # 32m
81 
82 DOWNLOADER = 'scrapy.core.downloader.Downloader'
83 
84 DOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory'
85 DOWNLOADER_CLIENTCONTEXTFACTORY = 'scrapy.core.downloader.contextfactory.ScrapyClientContextFactory'
86 DOWNLOADER_CLIENT_TLS_METHOD = 'TLS' # Use highest TLS/SSL protocol version supported by the platform,
87                                      # also allowing negotiation
88 
89 DOWNLOADER_MIDDLEWARES = {}
90 
91 DOWNLOADER_MIDDLEWARES_BASE = {
92     # Engine side
93     'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100,
94     'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300,
95     'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350,
96     'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 400,
97     'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 500,
98     'scrapy.downloadermiddlewares.retry.RetryMiddleware': 550,
99     'scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware': 560,
100     'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware': 580,
101     'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 590,
102     'scrapy.downloadermiddlewares.redirect.RedirectMiddleware': 600,
103     'scrapy.downloadermiddlewares.cookies.CookiesMiddleware': 700,
104     'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 750,
105     'scrapy.downloadermiddlewares.stats.DownloaderStats': 850,
106     'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware': 900,
107     # Downloader side
108 }
109 
110 DOWNLOADER_STATS = True
111 
112 DUPEFILTER_CLASS = 'scrapy.dupefilters.RFPDupeFilter'
113 
114 try:
115     EDITOR = os.environ['EDITOR']
116 except KeyError:
117     if sys.platform == 'win32':
118         EDITOR = '%s -m idlelib.idle'
119     else:
120         EDITOR = 'vi'
121 
122 EXTENSIONS = {}
123 
124 EXTENSIONS_BASE = {
125     'scrapy.extensions.corestats.CoreStats': 0,
126     'scrapy.extensions.telnet.TelnetConsole': 0,
127     'scrapy.extensions.memusage.MemoryUsage': 0,
128     'scrapy.extensions.memdebug.MemoryDebugger': 0,
129     'scrapy.extensions.closespider.CloseSpider': 0,
130     'scrapy.extensions.feedexport.FeedExporter': 0,
131     'scrapy.extensions.logstats.LogStats': 0,
132     'scrapy.extensions.spiderstate.SpiderState': 0,
133     'scrapy.extensions.throttle.AutoThrottle': 0,
134 }
135 
136 FEED_TEMPDIR = None
137 FEED_URI = None
138 FEED_URI_PARAMS = None  # a function to extend uri arguments
139 FEED_FORMAT = 'jsonlines'
140 FEED_STORE_EMPTY = False
141 FEED_EXPORT_ENCODING = None
142 FEED_EXPORT_FIELDS = None
143 FEED_STORAGES = {}
144 FEED_STORAGES_BASE = {
145     '': 'scrapy.extensions.feedexport.FileFeedStorage',
146     'file': 'scrapy.extensions.feedexport.FileFeedStorage',
147     'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage',
148     's3': 'scrapy.extensions.feedexport.S3FeedStorage',
149     'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',
150 }
151 FEED_EXPORTERS = {}
152 FEED_EXPORTERS_BASE = {
153     'json': 'scrapy.exporters.JsonItemExporter',
154     'jsonlines': 'scrapy.exporters.JsonLinesItemExporter',
155     'jl': 'scrapy.exporters.JsonLinesItemExporter',
156     'csv': 'scrapy.exporters.CsvItemExporter',
157     'xml': 'scrapy.exporters.XmlItemExporter',
158     'marshal': 'scrapy.exporters.MarshalItemExporter',
159     'pickle': 'scrapy.exporters.PickleItemExporter',
160 }
161 
162 FILES_STORE_S3_ACL = 'private'
163 
164 HTTPCACHE_ENABLED = False
165 HTTPCACHE_DIR = 'httpcache'
166 HTTPCACHE_IGNORE_MISSING = False
167 HTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'
168 HTTPCACHE_EXPIRATION_SECS = 0
169 HTTPCACHE_ALWAYS_STORE = False
170 HTTPCACHE_IGNORE_HTTP_CODES = []
171 HTTPCACHE_IGNORE_SCHEMES = ['file']
172 HTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS = []
173 HTTPCACHE_DBM_MODULE = 'anydbm' if six.PY2 else 'dbm'
174 HTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy'
175 HTTPCACHE_GZIP = False
176 
177 HTTPPROXY_AUTH_ENCODING = 'latin-1'
178 
179 IMAGES_STORE_S3_ACL = 'private'
180 
181 ITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager'
182 
183 ITEM_PIPELINES = {}
184 ITEM_PIPELINES_BASE = {}
185 
186 LOG_ENABLED = True
187 LOG_ENCODING = 'utf-8'
188 LOG_FORMATTER = 'scrapy.logformatter.LogFormatter'
189 LOG_FORMAT = '%(asctime)s [%(name)s] %(levelname)s: %(message)s'
190 LOG_DATEFORMAT = '%Y-%m-%d %H:%M:%S'
191 LOG_STDOUT = False
192 LOG_LEVEL = 'DEBUG'
193 LOG_FILE = None
194 LOG_SHORT_NAMES = False
195 
196 SCHEDULER_DEBUG = False
197 
198 LOGSTATS_INTERVAL = 60.0
199 
200 MAIL_HOST = 'localhost'
201 MAIL_PORT = 25
202 MAIL_FROM = 'scrapy@localhost'
203 MAIL_PASS = None
204 MAIL_USER = None
205 
206 MEMDEBUG_ENABLED = False        # enable memory debugging
207 MEMDEBUG_NOTIFY = []            # send memory debugging report by mail at engine shutdown
208 
209 MEMUSAGE_CHECK_INTERVAL_SECONDS = 60.0
210 MEMUSAGE_ENABLED = False
211 MEMUSAGE_LIMIT_MB = 0
212 MEMUSAGE_NOTIFY_MAIL = []
213 MEMUSAGE_REPORT = False
214 MEMUSAGE_WARNING_MB = 0
215 
216 METAREFRESH_ENABLED = True
217 METAREFRESH_MAXDELAY = 100
218 
219 NEWSPIDER_MODULE = ''
220 
221 RANDOMIZE_DOWNLOAD_DELAY = True
222 
223 REACTOR_THREADPOOL_MAXSIZE = 10
224 
225 REDIRECT_ENABLED = True
226 REDIRECT_MAX_TIMES = 20  # uses Firefox default setting
227 REDIRECT_PRIORITY_ADJUST = +2
228 
229 REFERER_ENABLED = True
230 
231 RETRY_ENABLED = True
232 RETRY_TIMES = 2  # initial response + 2 retries = 3 requests
233 RETRY_HTTP_CODES = [500, 502, 503, 504, 408]
234 RETRY_PRIORITY_ADJUST = -1
235 
236 ROBOTSTXT_OBEY = False
237 
238 SCHEDULER = 'scrapy.core.scheduler.Scheduler'
239 SCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleLifoDiskQueue'
240 SCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.LifoMemoryQueue'
241 SCHEDULER_PRIORITY_QUEUE = 'queuelib.PriorityQueue'
242 
243 SPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader'
244 
245 SPIDER_MIDDLEWARES = {}
246 
247 SPIDER_MIDDLEWARES_BASE = {
248     # Engine side
249     'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware': 50,
250     'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': 500,
251     'scrapy.spidermiddlewares.referer.RefererMiddleware': 700,
252     'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware': 800,
253     'scrapy.spidermiddlewares.depth.DepthMiddleware': 900,
254     # Spider side
255 }
256 
257 SPIDER_MODULES = []
258 
259 STATS_CLASS = 'scrapy.statscollectors.MemoryStatsCollector'
260 STATS_DUMP = True
261 
262 STATSMAILER_RCPTS = []
263 
264 TEMPLATES_DIR = abspath(join(dirname(__file__), '..', 'templates'))
265 
266 URLLENGTH_LIMIT = 2083
267 
268 USER_AGENT = 'Scrapy/%s (+http://scrapy.org)' % import_module('scrapy').__version__
269 
270 TELNETCONSOLE_ENABLED = 1
271 TELNETCONSOLE_PORT = [6023, 6073]
272 TELNETCONSOLE_HOST = '127.0.0.1'
273 
274 SPIDER_CONTRACTS = {}
275 SPIDER_CONTRACTS_BASE = {
276     'scrapy.contracts.default.UrlContract': 1,
277     'scrapy.contracts.default.ReturnsContract': 2,
278     'scrapy.contracts.default.ScrapesContract': 3,
279 }
280
```

Path: `scrapy/downloadermiddlewares/httpproxy.py`
Content:
```
1 import base64
2 from six.moves.urllib.request import getproxies, proxy_bypass
3 from six.moves.urllib.parse import unquote
4 try:
5     from urllib2 import _parse_proxy
6 except ImportError:
7     from urllib.request import _parse_proxy
8 from six.moves.urllib.parse import urlunparse
9 
10 from scrapy.utils.httpobj import urlparse_cached
11 from scrapy.exceptions import NotConfigured
12 from scrapy.utils.python import to_bytes
13 
14 
15 class HttpProxyMiddleware(object):
16 
17     def __init__(self, auth_encoding='latin-1'):
18         self.auth_encoding = auth_encoding
19         self.proxies = {}
20         for type, url in getproxies().items():
21             self.proxies[type] = self._get_proxy(url, type)
22 
23         if not self.proxies:
24             raise NotConfigured
25 
26     @classmethod
27     def from_crawler(cls, crawler):
28         auth_encoding = crawler.settings.get('HTTPPROXY_AUTH_ENCODING')
29         return cls(auth_encoding)
30 
31     def _get_proxy(self, url, orig_type):
32         proxy_type, user, password, hostport = _parse_proxy(url)
33         proxy_url = urlunparse((proxy_type or orig_type, hostport, '', '', '', ''))
34 
35         if user:
36             user_pass = to_bytes(
37                 '%s:%s' % (unquote(user), unquote(password)),
38                 encoding=self.auth_encoding)
39             creds = base64.b64encode(user_pass).strip()
40         else:
41             creds = None
42 
43         return creds, proxy_url
44 
45     def process_request(self, request, spider):
46         # ignore if proxy is already set
47         if 'proxy' in request.meta:
48             return
49 
50         parsed = urlparse_cached(request)
51         scheme = parsed.scheme
52 
53         # 'no_proxy' is only supported by http schemes
54         if scheme in ('http', 'https') and proxy_bypass(parsed.hostname):
55             return
56 
57         if scheme in self.proxies:
58             self._set_proxy(request, scheme)
59 
60     def _set_proxy(self, request, scheme):
61         creds, proxy = self.proxies[scheme]
62         request.meta['proxy'] = proxy
63         if creds:
64             request.headers['Proxy-Authorization'] = b'Basic ' + creds
65
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py
--- a/scrapy/downloadermiddlewares/httpproxy.py
+++ b/scrapy/downloadermiddlewares/httpproxy.py
@@ -20,23 +20,25 @@
         for type, url in getproxies().items():
             self.proxies[type] = self._get_proxy(url, type)
 
-        if not self.proxies:
-            raise NotConfigured
-
     @classmethod
     def from_crawler(cls, crawler):
+        if not crawler.settings.getbool('HTTPPROXY_ENABLED'):
+            raise NotConfigured
         auth_encoding = crawler.settings.get('HTTPPROXY_AUTH_ENCODING')
         return cls(auth_encoding)
 
+    def _basic_auth_header(self, username, password):
+        user_pass = to_bytes(
+            '%s:%s' % (unquote(username), unquote(password)),
+            encoding=self.auth_encoding)
+        return base64.b64encode(user_pass).strip()
+
     def _get_proxy(self, url, orig_type):
         proxy_type, user, password, hostport = _parse_proxy(url)
         proxy_url = urlunparse((proxy_type or orig_type, hostport, '', '', '', ''))
 
         if user:
-            user_pass = to_bytes(
-                '%s:%s' % (unquote(user), unquote(password)),
-                encoding=self.auth_encoding)
-            creds = base64.b64encode(user_pass).strip()
+            creds = self._basic_auth_header(user, password)
         else:
             creds = None
 
@@ -45,6 +47,15 @@
     def process_request(self, request, spider):
         # ignore if proxy is already set
         if 'proxy' in request.meta:
+            if request.meta['proxy'] is None:
+                return
+            # extract credentials if present
+            creds, proxy_url = self._get_proxy(request.meta['proxy'], '')
+            request.meta['proxy'] = proxy_url
+            if creds and not request.headers.get('Proxy-Authorization'):
+                request.headers['Proxy-Authorization'] = b'Basic ' + creds
+            return
+        elif not self.proxies:
             return
 
         parsed = urlparse_cached(request)
diff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py
--- a/scrapy/settings/default_settings.py
+++ b/scrapy/settings/default_settings.py
@@ -174,6 +174,7 @@
 HTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy'
 HTTPCACHE_GZIP = False
 
+HTTPPROXY_ENABLED = True
 HTTPPROXY_AUTH_ENCODING = 'latin-1'
 
 IMAGES_STORE_S3_ACL = 'private'

{"golden_diff": "diff --git a/scrapy/downloadermiddlewares/httpproxy.py b/scrapy/downloadermiddlewares/httpproxy.py\n--- a/scrapy/downloadermiddlewares/httpproxy.py\n+++ b/scrapy/downloadermiddlewares/httpproxy.py\n@@ -20,23 +20,25 @@\n         for type, url in getproxies().items():\n             self.proxies[type] = self._get_proxy(url, type)\n \n-        if not self.proxies:\n-            raise NotConfigured\n-\n     @classmethod\n     def from_crawler(cls, crawler):\n+        if not crawler.settings.getbool('HTTPPROXY_ENABLED'):\n+            raise NotConfigured\n         auth_encoding = crawler.settings.get('HTTPPROXY_AUTH_ENCODING')\n         return cls(auth_encoding)\n \n+    def _basic_auth_header(self, username, password):\n+        user_pass = to_bytes(\n+            '%s:%s' % (unquote(username), unquote(password)),\n+            encoding=self.auth_encoding)\n+        return base64.b64encode(user_pass).strip()\n+\n     def _get_proxy(self, url, orig_type):\n         proxy_type, user, password, hostport = _parse_proxy(url)\n         proxy_url = urlunparse((proxy_type or orig_type, hostport, '', '', '', ''))\n \n         if user:\n-            user_pass = to_bytes(\n-                '%s:%s' % (unquote(user), unquote(password)),\n-                encoding=self.auth_encoding)\n-            creds = base64.b64encode(user_pass).strip()\n+            creds = self._basic_auth_header(user, password)\n         else:\n             creds = None\n \n@@ -45,6 +47,15 @@\n     def process_request(self, request, spider):\n         # ignore if proxy is already set\n         if 'proxy' in request.meta:\n+            if request.meta['proxy'] is None:\n+                return\n+            # extract credentials if present\n+            creds, proxy_url = self._get_proxy(request.meta['proxy'], '')\n+            request.meta['proxy'] = proxy_url\n+            if creds and not request.headers.get('Proxy-Authorization'):\n+                request.headers['Proxy-Authorization'] = b'Basic ' + creds\n+            return\n+        elif not self.proxies:\n             return\n \n         parsed = urlparse_cached(request)\ndiff --git a/scrapy/settings/default_settings.py b/scrapy/settings/default_settings.py\n--- a/scrapy/settings/default_settings.py\n+++ b/scrapy/settings/default_settings.py\n@@ -174,6 +174,7 @@\n HTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy'\n HTTPCACHE_GZIP = False\n \n+HTTPPROXY_ENABLED = True\n HTTPPROXY_AUTH_ENCODING = 'latin-1'\n \n IMAGES_STORE_S3_ACL = 'private'\n", "issue": "Scrapy ignores proxy credentials when using \"proxy\" meta key\nCode `yield Request(link, meta={'proxy': 'http://user:password@ip:port\u2019})` ignores user:password. \r\nProblem is solved by using header \"Proxy-Authorization\" with base64, but it is better to implement it inside Scrapy.\n", "before_files": [{"content": "\"\"\"\nThis module contains the default values for all settings used by Scrapy.\n\nFor more information about these settings you can read the settings\ndocumentation in docs/topics/settings.rst\n\nScrapy developers, if you add a setting here remember to:\n\n* add it in alphabetical order\n* group similar settings without leaving blank lines\n* add its documentation to the available settings documentation\n  (docs/topics/settings.rst)\n\n\"\"\"\n\nimport os\nimport sys\nfrom importlib import import_module\nfrom os.path import join, abspath, dirname\n\nimport six\n\nAJAXCRAWL_ENABLED = False\n\nAUTOTHROTTLE_ENABLED = False\nAUTOTHROTTLE_DEBUG = False\nAUTOTHROTTLE_MAX_DELAY = 60.0\nAUTOTHROTTLE_START_DELAY = 5.0\nAUTOTHROTTLE_TARGET_CONCURRENCY = 1.0\n\nBOT_NAME = 'scrapybot'\n\nCLOSESPIDER_TIMEOUT = 0\nCLOSESPIDER_PAGECOUNT = 0\nCLOSESPIDER_ITEMCOUNT = 0\nCLOSESPIDER_ERRORCOUNT = 0\n\nCOMMANDS_MODULE = ''\n\nCOMPRESSION_ENABLED = True\n\nCONCURRENT_ITEMS = 100\n\nCONCURRENT_REQUESTS = 16\nCONCURRENT_REQUESTS_PER_DOMAIN = 8\nCONCURRENT_REQUESTS_PER_IP = 0\n\nCOOKIES_ENABLED = True\nCOOKIES_DEBUG = False\n\nDEFAULT_ITEM_CLASS = 'scrapy.item.Item'\n\nDEFAULT_REQUEST_HEADERS = {\n    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',\n    'Accept-Language': 'en',\n}\n\nDEPTH_LIMIT = 0\nDEPTH_STATS = True\nDEPTH_PRIORITY = 0\n\nDNSCACHE_ENABLED = True\nDNSCACHE_SIZE = 10000\nDNS_TIMEOUT = 60\n\nDOWNLOAD_DELAY = 0\n\nDOWNLOAD_HANDLERS = {}\nDOWNLOAD_HANDLERS_BASE = {\n    'file': 'scrapy.core.downloader.handlers.file.FileDownloadHandler',\n    'http': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',\n    'https': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',\n    's3': 'scrapy.core.downloader.handlers.s3.S3DownloadHandler',\n    'ftp': 'scrapy.core.downloader.handlers.ftp.FTPDownloadHandler',\n}\n\nDOWNLOAD_TIMEOUT = 180      # 3mins\n\nDOWNLOAD_MAXSIZE = 1024*1024*1024   # 1024m\nDOWNLOAD_WARNSIZE = 32*1024*1024    # 32m\n\nDOWNLOADER = 'scrapy.core.downloader.Downloader'\n\nDOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory'\nDOWNLOADER_CLIENTCONTEXTFACTORY = 'scrapy.core.downloader.contextfactory.ScrapyClientContextFactory'\nDOWNLOADER_CLIENT_TLS_METHOD = 'TLS' # Use highest TLS/SSL protocol version supported by the platform,\n                                     # also allowing negotiation\n\nDOWNLOADER_MIDDLEWARES = {}\n\nDOWNLOADER_MIDDLEWARES_BASE = {\n    # Engine side\n    'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100,\n    'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300,\n    'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350,\n    'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 400,\n    'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 500,\n    'scrapy.downloadermiddlewares.retry.RetryMiddleware': 550,\n    'scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware': 560,\n    'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware': 580,\n    'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 590,\n    'scrapy.downloadermiddlewares.redirect.RedirectMiddleware': 600,\n    'scrapy.downloadermiddlewares.cookies.CookiesMiddleware': 700,\n    'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 750,\n    'scrapy.downloadermiddlewares.stats.DownloaderStats': 850,\n    'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware': 900,\n    # Downloader side\n}\n\nDOWNLOADER_STATS = True\n\nDUPEFILTER_CLASS = 'scrapy.dupefilters.RFPDupeFilter'\n\ntry:\n    EDITOR = os.environ['EDITOR']\nexcept KeyError:\n    if sys.platform == 'win32':\n        EDITOR = '%s -m idlelib.idle'\n    else:\n        EDITOR = 'vi'\n\nEXTENSIONS = {}\n\nEXTENSIONS_BASE = {\n    'scrapy.extensions.corestats.CoreStats': 0,\n    'scrapy.extensions.telnet.TelnetConsole': 0,\n    'scrapy.extensions.memusage.MemoryUsage': 0,\n    'scrapy.extensions.memdebug.MemoryDebugger': 0,\n    'scrapy.extensions.closespider.CloseSpider': 0,\n    'scrapy.extensions.feedexport.FeedExporter': 0,\n    'scrapy.extensions.logstats.LogStats': 0,\n    'scrapy.extensions.spiderstate.SpiderState': 0,\n    'scrapy.extensions.throttle.AutoThrottle': 0,\n}\n\nFEED_TEMPDIR = None\nFEED_URI = None\nFEED_URI_PARAMS = None  # a function to extend uri arguments\nFEED_FORMAT = 'jsonlines'\nFEED_STORE_EMPTY = False\nFEED_EXPORT_ENCODING = None\nFEED_EXPORT_FIELDS = None\nFEED_STORAGES = {}\nFEED_STORAGES_BASE = {\n    '': 'scrapy.extensions.feedexport.FileFeedStorage',\n    'file': 'scrapy.extensions.feedexport.FileFeedStorage',\n    'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage',\n    's3': 'scrapy.extensions.feedexport.S3FeedStorage',\n    'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',\n}\nFEED_EXPORTERS = {}\nFEED_EXPORTERS_BASE = {\n    'json': 'scrapy.exporters.JsonItemExporter',\n    'jsonlines': 'scrapy.exporters.JsonLinesItemExporter',\n    'jl': 'scrapy.exporters.JsonLinesItemExporter',\n    'csv': 'scrapy.exporters.CsvItemExporter',\n    'xml': 'scrapy.exporters.XmlItemExporter',\n    'marshal': 'scrapy.exporters.MarshalItemExporter',\n    'pickle': 'scrapy.exporters.PickleItemExporter',\n}\n\nFILES_STORE_S3_ACL = 'private'\n\nHTTPCACHE_ENABLED = False\nHTTPCACHE_DIR = 'httpcache'\nHTTPCACHE_IGNORE_MISSING = False\nHTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'\nHTTPCACHE_EXPIRATION_SECS = 0\nHTTPCACHE_ALWAYS_STORE = False\nHTTPCACHE_IGNORE_HTTP_CODES = []\nHTTPCACHE_IGNORE_SCHEMES = ['file']\nHTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS = []\nHTTPCACHE_DBM_MODULE = 'anydbm' if six.PY2 else 'dbm'\nHTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy'\nHTTPCACHE_GZIP = False\n\nHTTPPROXY_AUTH_ENCODING = 'latin-1'\n\nIMAGES_STORE_S3_ACL = 'private'\n\nITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager'\n\nITEM_PIPELINES = {}\nITEM_PIPELINES_BASE = {}\n\nLOG_ENABLED = True\nLOG_ENCODING = 'utf-8'\nLOG_FORMATTER = 'scrapy.logformatter.LogFormatter'\nLOG_FORMAT = '%(asctime)s [%(name)s] %(levelname)s: %(message)s'\nLOG_DATEFORMAT = '%Y-%m-%d %H:%M:%S'\nLOG_STDOUT = False\nLOG_LEVEL = 'DEBUG'\nLOG_FILE = None\nLOG_SHORT_NAMES = False\n\nSCHEDULER_DEBUG = False\n\nLOGSTATS_INTERVAL = 60.0\n\nMAIL_HOST = 'localhost'\nMAIL_PORT = 25\nMAIL_FROM = 'scrapy@localhost'\nMAIL_PASS = None\nMAIL_USER = None\n\nMEMDEBUG_ENABLED = False        # enable memory debugging\nMEMDEBUG_NOTIFY = []            # send memory debugging report by mail at engine shutdown\n\nMEMUSAGE_CHECK_INTERVAL_SECONDS = 60.0\nMEMUSAGE_ENABLED = False\nMEMUSAGE_LIMIT_MB = 0\nMEMUSAGE_NOTIFY_MAIL = []\nMEMUSAGE_REPORT = False\nMEMUSAGE_WARNING_MB = 0\n\nMETAREFRESH_ENABLED = True\nMETAREFRESH_MAXDELAY = 100\n\nNEWSPIDER_MODULE = ''\n\nRANDOMIZE_DOWNLOAD_DELAY = True\n\nREACTOR_THREADPOOL_MAXSIZE = 10\n\nREDIRECT_ENABLED = True\nREDIRECT_MAX_TIMES = 20  # uses Firefox default setting\nREDIRECT_PRIORITY_ADJUST = +2\n\nREFERER_ENABLED = True\n\nRETRY_ENABLED = True\nRETRY_TIMES = 2  # initial response + 2 retries = 3 requests\nRETRY_HTTP_CODES = [500, 502, 503, 504, 408]\nRETRY_PRIORITY_ADJUST = -1\n\nROBOTSTXT_OBEY = False\n\nSCHEDULER = 'scrapy.core.scheduler.Scheduler'\nSCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleLifoDiskQueue'\nSCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.LifoMemoryQueue'\nSCHEDULER_PRIORITY_QUEUE = 'queuelib.PriorityQueue'\n\nSPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader'\n\nSPIDER_MIDDLEWARES = {}\n\nSPIDER_MIDDLEWARES_BASE = {\n    # Engine side\n    'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware': 50,\n    'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': 500,\n    'scrapy.spidermiddlewares.referer.RefererMiddleware': 700,\n    'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware': 800,\n    'scrapy.spidermiddlewares.depth.DepthMiddleware': 900,\n    # Spider side\n}\n\nSPIDER_MODULES = []\n\nSTATS_CLASS = 'scrapy.statscollectors.MemoryStatsCollector'\nSTATS_DUMP = True\n\nSTATSMAILER_RCPTS = []\n\nTEMPLATES_DIR = abspath(join(dirname(__file__), '..', 'templates'))\n\nURLLENGTH_LIMIT = 2083\n\nUSER_AGENT = 'Scrapy/%s (+http://scrapy.org)' % import_module('scrapy').__version__\n\nTELNETCONSOLE_ENABLED = 1\nTELNETCONSOLE_PORT = [6023, 6073]\nTELNETCONSOLE_HOST = '127.0.0.1'\n\nSPIDER_CONTRACTS = {}\nSPIDER_CONTRACTS_BASE = {\n    'scrapy.contracts.default.UrlContract': 1,\n    'scrapy.contracts.default.ReturnsContract': 2,\n    'scrapy.contracts.default.ScrapesContract': 3,\n}\n", "path": "scrapy/settings/default_settings.py"}, {"content": "import base64\nfrom six.moves.urllib.request import getproxies, proxy_bypass\nfrom six.moves.urllib.parse import unquote\ntry:\n    from urllib2 import _parse_proxy\nexcept ImportError:\n    from urllib.request import _parse_proxy\nfrom six.moves.urllib.parse import urlunparse\n\nfrom scrapy.utils.httpobj import urlparse_cached\nfrom scrapy.exceptions import NotConfigured\nfrom scrapy.utils.python import to_bytes\n\n\nclass HttpProxyMiddleware(object):\n\n    def __init__(self, auth_encoding='latin-1'):\n        self.auth_encoding = auth_encoding\n        self.proxies = {}\n        for type, url in getproxies().items():\n            self.proxies[type] = self._get_proxy(url, type)\n\n        if not self.proxies:\n            raise NotConfigured\n\n    @classmethod\n    def from_crawler(cls, crawler):\n        auth_encoding = crawler.settings.get('HTTPPROXY_AUTH_ENCODING')\n        return cls(auth_encoding)\n\n    def _get_proxy(self, url, orig_type):\n        proxy_type, user, password, hostport = _parse_proxy(url)\n        proxy_url = urlunparse((proxy_type or orig_type, hostport, '', '', '', ''))\n\n        if user:\n            user_pass = to_bytes(\n                '%s:%s' % (unquote(user), unquote(password)),\n                encoding=self.auth_encoding)\n            creds = base64.b64encode(user_pass).strip()\n        else:\n            creds = None\n\n        return creds, proxy_url\n\n    def process_request(self, request, spider):\n        # ignore if proxy is already set\n        if 'proxy' in request.meta:\n            return\n\n        parsed = urlparse_cached(request)\n        scheme = parsed.scheme\n\n        # 'no_proxy' is only supported by http schemes\n        if scheme in ('http', 'https') and proxy_bypass(parsed.hostname):\n            return\n\n        if scheme in self.proxies:\n            self._set_proxy(request, scheme)\n\n    def _set_proxy(self, request, scheme):\n        creds, proxy = self.proxies[scheme]\n        request.meta['proxy'] = proxy\n        if creds:\n            request.headers['Proxy-Authorization'] = b'Basic ' + creds\n", "path": "scrapy/downloadermiddlewares/httpproxy.py"}], "after_files": [{"content": "\"\"\"\nThis module contains the default values for all settings used by Scrapy.\n\nFor more information about these settings you can read the settings\ndocumentation in docs/topics/settings.rst\n\nScrapy developers, if you add a setting here remember to:\n\n* add it in alphabetical order\n* group similar settings without leaving blank lines\n* add its documentation to the available settings documentation\n  (docs/topics/settings.rst)\n\n\"\"\"\n\nimport os\nimport sys\nfrom importlib import import_module\nfrom os.path import join, abspath, dirname\n\nimport six\n\nAJAXCRAWL_ENABLED = False\n\nAUTOTHROTTLE_ENABLED = False\nAUTOTHROTTLE_DEBUG = False\nAUTOTHROTTLE_MAX_DELAY = 60.0\nAUTOTHROTTLE_START_DELAY = 5.0\nAUTOTHROTTLE_TARGET_CONCURRENCY = 1.0\n\nBOT_NAME = 'scrapybot'\n\nCLOSESPIDER_TIMEOUT = 0\nCLOSESPIDER_PAGECOUNT = 0\nCLOSESPIDER_ITEMCOUNT = 0\nCLOSESPIDER_ERRORCOUNT = 0\n\nCOMMANDS_MODULE = ''\n\nCOMPRESSION_ENABLED = True\n\nCONCURRENT_ITEMS = 100\n\nCONCURRENT_REQUESTS = 16\nCONCURRENT_REQUESTS_PER_DOMAIN = 8\nCONCURRENT_REQUESTS_PER_IP = 0\n\nCOOKIES_ENABLED = True\nCOOKIES_DEBUG = False\n\nDEFAULT_ITEM_CLASS = 'scrapy.item.Item'\n\nDEFAULT_REQUEST_HEADERS = {\n    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',\n    'Accept-Language': 'en',\n}\n\nDEPTH_LIMIT = 0\nDEPTH_STATS = True\nDEPTH_PRIORITY = 0\n\nDNSCACHE_ENABLED = True\nDNSCACHE_SIZE = 10000\nDNS_TIMEOUT = 60\n\nDOWNLOAD_DELAY = 0\n\nDOWNLOAD_HANDLERS = {}\nDOWNLOAD_HANDLERS_BASE = {\n    'file': 'scrapy.core.downloader.handlers.file.FileDownloadHandler',\n    'http': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',\n    'https': 'scrapy.core.downloader.handlers.http.HTTPDownloadHandler',\n    's3': 'scrapy.core.downloader.handlers.s3.S3DownloadHandler',\n    'ftp': 'scrapy.core.downloader.handlers.ftp.FTPDownloadHandler',\n}\n\nDOWNLOAD_TIMEOUT = 180      # 3mins\n\nDOWNLOAD_MAXSIZE = 1024*1024*1024   # 1024m\nDOWNLOAD_WARNSIZE = 32*1024*1024    # 32m\n\nDOWNLOADER = 'scrapy.core.downloader.Downloader'\n\nDOWNLOADER_HTTPCLIENTFACTORY = 'scrapy.core.downloader.webclient.ScrapyHTTPClientFactory'\nDOWNLOADER_CLIENTCONTEXTFACTORY = 'scrapy.core.downloader.contextfactory.ScrapyClientContextFactory'\nDOWNLOADER_CLIENT_TLS_METHOD = 'TLS' # Use highest TLS/SSL protocol version supported by the platform,\n                                     # also allowing negotiation\n\nDOWNLOADER_MIDDLEWARES = {}\n\nDOWNLOADER_MIDDLEWARES_BASE = {\n    # Engine side\n    'scrapy.downloadermiddlewares.robotstxt.RobotsTxtMiddleware': 100,\n    'scrapy.downloadermiddlewares.httpauth.HttpAuthMiddleware': 300,\n    'scrapy.downloadermiddlewares.downloadtimeout.DownloadTimeoutMiddleware': 350,\n    'scrapy.downloadermiddlewares.defaultheaders.DefaultHeadersMiddleware': 400,\n    'scrapy.downloadermiddlewares.useragent.UserAgentMiddleware': 500,\n    'scrapy.downloadermiddlewares.retry.RetryMiddleware': 550,\n    'scrapy.downloadermiddlewares.ajaxcrawl.AjaxCrawlMiddleware': 560,\n    'scrapy.downloadermiddlewares.redirect.MetaRefreshMiddleware': 580,\n    'scrapy.downloadermiddlewares.httpcompression.HttpCompressionMiddleware': 590,\n    'scrapy.downloadermiddlewares.redirect.RedirectMiddleware': 600,\n    'scrapy.downloadermiddlewares.cookies.CookiesMiddleware': 700,\n    'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 750,\n    'scrapy.downloadermiddlewares.stats.DownloaderStats': 850,\n    'scrapy.downloadermiddlewares.httpcache.HttpCacheMiddleware': 900,\n    # Downloader side\n}\n\nDOWNLOADER_STATS = True\n\nDUPEFILTER_CLASS = 'scrapy.dupefilters.RFPDupeFilter'\n\ntry:\n    EDITOR = os.environ['EDITOR']\nexcept KeyError:\n    if sys.platform == 'win32':\n        EDITOR = '%s -m idlelib.idle'\n    else:\n        EDITOR = 'vi'\n\nEXTENSIONS = {}\n\nEXTENSIONS_BASE = {\n    'scrapy.extensions.corestats.CoreStats': 0,\n    'scrapy.extensions.telnet.TelnetConsole': 0,\n    'scrapy.extensions.memusage.MemoryUsage': 0,\n    'scrapy.extensions.memdebug.MemoryDebugger': 0,\n    'scrapy.extensions.closespider.CloseSpider': 0,\n    'scrapy.extensions.feedexport.FeedExporter': 0,\n    'scrapy.extensions.logstats.LogStats': 0,\n    'scrapy.extensions.spiderstate.SpiderState': 0,\n    'scrapy.extensions.throttle.AutoThrottle': 0,\n}\n\nFEED_TEMPDIR = None\nFEED_URI = None\nFEED_URI_PARAMS = None  # a function to extend uri arguments\nFEED_FORMAT = 'jsonlines'\nFEED_STORE_EMPTY = False\nFEED_EXPORT_ENCODING = None\nFEED_EXPORT_FIELDS = None\nFEED_STORAGES = {}\nFEED_STORAGES_BASE = {\n    '': 'scrapy.extensions.feedexport.FileFeedStorage',\n    'file': 'scrapy.extensions.feedexport.FileFeedStorage',\n    'stdout': 'scrapy.extensions.feedexport.StdoutFeedStorage',\n    's3': 'scrapy.extensions.feedexport.S3FeedStorage',\n    'ftp': 'scrapy.extensions.feedexport.FTPFeedStorage',\n}\nFEED_EXPORTERS = {}\nFEED_EXPORTERS_BASE = {\n    'json': 'scrapy.exporters.JsonItemExporter',\n    'jsonlines': 'scrapy.exporters.JsonLinesItemExporter',\n    'jl': 'scrapy.exporters.JsonLinesItemExporter',\n    'csv': 'scrapy.exporters.CsvItemExporter',\n    'xml': 'scrapy.exporters.XmlItemExporter',\n    'marshal': 'scrapy.exporters.MarshalItemExporter',\n    'pickle': 'scrapy.exporters.PickleItemExporter',\n}\n\nFILES_STORE_S3_ACL = 'private'\n\nHTTPCACHE_ENABLED = False\nHTTPCACHE_DIR = 'httpcache'\nHTTPCACHE_IGNORE_MISSING = False\nHTTPCACHE_STORAGE = 'scrapy.extensions.httpcache.FilesystemCacheStorage'\nHTTPCACHE_EXPIRATION_SECS = 0\nHTTPCACHE_ALWAYS_STORE = False\nHTTPCACHE_IGNORE_HTTP_CODES = []\nHTTPCACHE_IGNORE_SCHEMES = ['file']\nHTTPCACHE_IGNORE_RESPONSE_CACHE_CONTROLS = []\nHTTPCACHE_DBM_MODULE = 'anydbm' if six.PY2 else 'dbm'\nHTTPCACHE_POLICY = 'scrapy.extensions.httpcache.DummyPolicy'\nHTTPCACHE_GZIP = False\n\nHTTPPROXY_ENABLED = True\nHTTPPROXY_AUTH_ENCODING = 'latin-1'\n\nIMAGES_STORE_S3_ACL = 'private'\n\nITEM_PROCESSOR = 'scrapy.pipelines.ItemPipelineManager'\n\nITEM_PIPELINES = {}\nITEM_PIPELINES_BASE = {}\n\nLOG_ENABLED = True\nLOG_ENCODING = 'utf-8'\nLOG_FORMATTER = 'scrapy.logformatter.LogFormatter'\nLOG_FORMAT = '%(asctime)s [%(name)s] %(levelname)s: %(message)s'\nLOG_DATEFORMAT = '%Y-%m-%d %H:%M:%S'\nLOG_STDOUT = False\nLOG_LEVEL = 'DEBUG'\nLOG_FILE = None\nLOG_SHORT_NAMES = False\n\nSCHEDULER_DEBUG = False\n\nLOGSTATS_INTERVAL = 60.0\n\nMAIL_HOST = 'localhost'\nMAIL_PORT = 25\nMAIL_FROM = 'scrapy@localhost'\nMAIL_PASS = None\nMAIL_USER = None\n\nMEMDEBUG_ENABLED = False        # enable memory debugging\nMEMDEBUG_NOTIFY = []            # send memory debugging report by mail at engine shutdown\n\nMEMUSAGE_CHECK_INTERVAL_SECONDS = 60.0\nMEMUSAGE_ENABLED = False\nMEMUSAGE_LIMIT_MB = 0\nMEMUSAGE_NOTIFY_MAIL = []\nMEMUSAGE_REPORT = False\nMEMUSAGE_WARNING_MB = 0\n\nMETAREFRESH_ENABLED = True\nMETAREFRESH_MAXDELAY = 100\n\nNEWSPIDER_MODULE = ''\n\nRANDOMIZE_DOWNLOAD_DELAY = True\n\nREACTOR_THREADPOOL_MAXSIZE = 10\n\nREDIRECT_ENABLED = True\nREDIRECT_MAX_TIMES = 20  # uses Firefox default setting\nREDIRECT_PRIORITY_ADJUST = +2\n\nREFERER_ENABLED = True\n\nRETRY_ENABLED = True\nRETRY_TIMES = 2  # initial response + 2 retries = 3 requests\nRETRY_HTTP_CODES = [500, 502, 503, 504, 408]\nRETRY_PRIORITY_ADJUST = -1\n\nROBOTSTXT_OBEY = False\n\nSCHEDULER = 'scrapy.core.scheduler.Scheduler'\nSCHEDULER_DISK_QUEUE = 'scrapy.squeues.PickleLifoDiskQueue'\nSCHEDULER_MEMORY_QUEUE = 'scrapy.squeues.LifoMemoryQueue'\nSCHEDULER_PRIORITY_QUEUE = 'queuelib.PriorityQueue'\n\nSPIDER_LOADER_CLASS = 'scrapy.spiderloader.SpiderLoader'\n\nSPIDER_MIDDLEWARES = {}\n\nSPIDER_MIDDLEWARES_BASE = {\n    # Engine side\n    'scrapy.spidermiddlewares.httperror.HttpErrorMiddleware': 50,\n    'scrapy.spidermiddlewares.offsite.OffsiteMiddleware': 500,\n    'scrapy.spidermiddlewares.referer.RefererMiddleware': 700,\n    'scrapy.spidermiddlewares.urllength.UrlLengthMiddleware': 800,\n    'scrapy.spidermiddlewares.depth.DepthMiddleware': 900,\n    # Spider side\n}\n\nSPIDER_MODULES = []\n\nSTATS_CLASS = 'scrapy.statscollectors.MemoryStatsCollector'\nSTATS_DUMP = True\n\nSTATSMAILER_RCPTS = []\n\nTEMPLATES_DIR = abspath(join(dirname(__file__), '..', 'templates'))\n\nURLLENGTH_LIMIT = 2083\n\nUSER_AGENT = 'Scrapy/%s (+http://scrapy.org)' % import_module('scrapy').__version__\n\nTELNETCONSOLE_ENABLED = 1\nTELNETCONSOLE_PORT = [6023, 6073]\nTELNETCONSOLE_HOST = '127.0.0.1'\n\nSPIDER_CONTRACTS = {}\nSPIDER_CONTRACTS_BASE = {\n    'scrapy.contracts.default.UrlContract': 1,\n    'scrapy.contracts.default.ReturnsContract': 2,\n    'scrapy.contracts.default.ScrapesContract': 3,\n}\n", "path": "scrapy/settings/default_settings.py"}, {"content": "import base64\nfrom six.moves.urllib.request import getproxies, proxy_bypass\nfrom six.moves.urllib.parse import unquote\ntry:\n    from urllib2 import _parse_proxy\nexcept ImportError:\n    from urllib.request import _parse_proxy\nfrom six.moves.urllib.parse import urlunparse\n\nfrom scrapy.utils.httpobj import urlparse_cached\nfrom scrapy.exceptions import NotConfigured\nfrom scrapy.utils.python import to_bytes\n\n\nclass HttpProxyMiddleware(object):\n\n    def __init__(self, auth_encoding='latin-1'):\n        self.auth_encoding = auth_encoding\n        self.proxies = {}\n        for type, url in getproxies().items():\n            self.proxies[type] = self._get_proxy(url, type)\n\n    @classmethod\n    def from_crawler(cls, crawler):\n        if not crawler.settings.getbool('HTTPPROXY_ENABLED'):\n            raise NotConfigured\n        auth_encoding = crawler.settings.get('HTTPPROXY_AUTH_ENCODING')\n        return cls(auth_encoding)\n\n    def _basic_auth_header(self, username, password):\n        user_pass = to_bytes(\n            '%s:%s' % (unquote(username), unquote(password)),\n            encoding=self.auth_encoding)\n        return base64.b64encode(user_pass).strip()\n\n    def _get_proxy(self, url, orig_type):\n        proxy_type, user, password, hostport = _parse_proxy(url)\n        proxy_url = urlunparse((proxy_type or orig_type, hostport, '', '', '', ''))\n\n        if user:\n            creds = self._basic_auth_header(user, password)\n        else:\n            creds = None\n\n        return creds, proxy_url\n\n    def process_request(self, request, spider):\n        # ignore if proxy is already set\n        if 'proxy' in request.meta:\n            if request.meta['proxy'] is None:\n                return\n            # extract credentials if present\n            creds, proxy_url = self._get_proxy(request.meta['proxy'], '')\n            request.meta['proxy'] = proxy_url\n            if creds and not request.headers.get('Proxy-Authorization'):\n                request.headers['Proxy-Authorization'] = b'Basic ' + creds\n            return\n        elif not self.proxies:\n            return\n\n        parsed = urlparse_cached(request)\n        scheme = parsed.scheme\n\n        # 'no_proxy' is only supported by http schemes\n        if scheme in ('http', 'https') and proxy_bypass(parsed.hostname):\n            return\n\n        if scheme in self.proxies:\n            self._set_proxy(request, scheme)\n\n    def _set_proxy(self, request, scheme):\n        creds, proxy = self.proxies[scheme]\n        request.meta['proxy'] = proxy\n        if creds:\n            request.headers['Proxy-Authorization'] = b'Basic ' + creds\n", "path": "scrapy/downloadermiddlewares/httpproxy.py"}]}