We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
SSL error
I have lot of errors with SSL websites.
For exemple, when I call : `scrapy shell https://subscribe.wsj.com/printpack/`

I have this error : 

```
2016-10-06 22:15:40 [scrapy] DEBUG: Telnet console listening on 127.0.0.1:6043
2016-10-06 22:15:40 [scrapy] INFO: Spider opened
2016-10-06 22:15:40 [scrapy] DEBUG: Retrying <GET https://subscribe.wsj.com/printpack/> (failed 1 times): [<twisted.python.failure.Failure OpenSSL.SSL.Error: [('SSL routines', 'SSL23_GET_SERVER_HELLO', 'sslv3 alert handshake failure')]>]
2016-10-06 22:15:40 [scrapy] DEBUG: Retrying <GET https://subscribe.wsj.com/printpack/> (failed 2 times): [<twisted.python.failure.Failure OpenSSL.SSL.Error: [('SSL routines', 'SSL23_GET_SERVER_HELLO', 'sslv3 alert handshake failure')]>]
2016-10-06 22:15:40 [scrapy] DEBUG: Gave up retrying <GET https://subscribe.wsj.com/printpack/> (failed 3 times): [<twisted.python.failure.Failure OpenSSL.SSL.Error: [('SSL routines', 'SSL23_GET_SERVER_HELLO', 'sslv3 alert handshake failure')]>]
Traceback (most recent call last):
  File "/usr/local/bin/scrapy", line 11, in <module>
    sys.exit(execute())
  File "/usr/local/lib/python2.7/dist-packages/scrapy/cmdline.py", line 142, in execute
    _run_print_help(parser, _run_command, cmd, args, opts)
  File "/usr/local/lib/python2.7/dist-packages/scrapy/cmdline.py", line 88, in _run_print_help
    func(*a, **kw)
  File "/usr/local/lib/python2.7/dist-packages/scrapy/cmdline.py", line 149, in _run_command
    cmd.run(args, opts)
  File "/usr/local/lib/python2.7/dist-packages/scrapy/commands/shell.py", line 71, in run
    shell.start(url=url)
  File "/usr/local/lib/python2.7/dist-packages/scrapy/shell.py", line 47, in start
    self.fetch(url, spider)
  File "/usr/local/lib/python2.7/dist-packages/scrapy/shell.py", line 112, in fetch
    reactor, self._schedule, request, spider)
  File "/usr/local/lib/python2.7/dist-packages/twisted/internet/threads.py", line 122, in blockingCallFromThread
    result.raiseException()
  File "<string>", line 2, in raiseException
twisted.web._newclient.ResponseNeverReceived: [<twisted.python.failure.Failure OpenSSL.SSL.Error: [('SSL routines', 'SSL23_GET_SERVER_HELLO', 'sslv3 alert handshake failure')]>]
```

How fix that ?


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `scrapy/core/downloader/tls.py`
Content:
```
1 import logging
2 from OpenSSL import SSL
3 
4 
5 logger = logging.getLogger(__name__)
6 
7 METHOD_SSLv3 = 'SSLv3'
8 METHOD_TLS = 'TLS'
9 METHOD_TLSv10 = 'TLSv1.0'
10 METHOD_TLSv11 = 'TLSv1.1'
11 METHOD_TLSv12 = 'TLSv1.2'
12 
13 openssl_methods = {
14     METHOD_TLS:    SSL.SSLv23_METHOD,                   # protocol negotiation (recommended)
15     METHOD_SSLv3:  SSL.SSLv3_METHOD,                    # SSL 3 (NOT recommended)
16     METHOD_TLSv10: SSL.TLSv1_METHOD,                    # TLS 1.0 only
17     METHOD_TLSv11: getattr(SSL, 'TLSv1_1_METHOD', 5),   # TLS 1.1 only
18     METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6),   # TLS 1.2 only
19 }
20 
21 # ClientTLSOptions requires a recent-enough version of Twisted
22 try:
23 
24     # taken from twisted/twisted/internet/_sslverify.py
25     try:
26         from OpenSSL.SSL import SSL_CB_HANDSHAKE_DONE, SSL_CB_HANDSHAKE_START
27     except ImportError:
28         SSL_CB_HANDSHAKE_START = 0x10
29         SSL_CB_HANDSHAKE_DONE = 0x20
30 
31     from twisted.internet._sslverify import (ClientTLSOptions,
32                                              _maybeSetHostNameIndication,
33                                              verifyHostname,
34                                              VerificationError)
35 
36     class ScrapyClientTLSOptions(ClientTLSOptions):
37         """
38         SSL Client connection creator ignoring certificate verification errors
39         (for genuinely invalid certificates or bugs in verification code).
40 
41         Same as Twisted's private _sslverify.ClientTLSOptions,
42         except that VerificationError and ValueError exceptions are caught,
43         so that the connection is not closed, only logging warnings.
44         """
45 
46         def _identityVerifyingInfoCallback(self, connection, where, ret):
47             if where & SSL_CB_HANDSHAKE_START:
48                 _maybeSetHostNameIndication(connection, self._hostnameBytes)
49             elif where & SSL_CB_HANDSHAKE_DONE:
50                 try:
51                     verifyHostname(connection, self._hostnameASCII)
52                 except VerificationError as e:
53                     logger.warning(
54                         'Remote certificate is not valid for hostname "{}"; {}'.format(
55                             self._hostnameASCII, e))
56 
57                 except ValueError as e:
58                     logger.warning(
59                         'Ignoring error while verifying certificate '
60                         'from host "{}" (exception: {})'.format(
61                             self._hostnameASCII, repr(e)))
62 
63 except ImportError:
64     # ImportError should not matter for older Twisted versions
65     # as the above is not used in the fallback ScrapyClientContextFactory
66     pass
67
```

Path: `scrapy/core/downloader/contextfactory.py`
Content:
```
1 from OpenSSL import SSL
2 from twisted.internet.ssl import ClientContextFactory
3 
4 try:
5 
6     from zope.interface.declarations import implementer
7 
8     # the following should be available from Twisted 14.0.0
9     from twisted.internet.ssl import (optionsForClientTLS,
10                                       CertificateOptions,
11                                       platformTrust)
12 
13     from twisted.web.client import BrowserLikePolicyForHTTPS
14     from twisted.web.iweb import IPolicyForHTTPS
15 
16     from scrapy.core.downloader.tls import ScrapyClientTLSOptions
17 
18 
19     @implementer(IPolicyForHTTPS)
20     class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):
21         """
22         Non-peer-certificate verifying HTTPS context factory
23 
24         Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD)
25         which allows TLS protocol negotiation
26 
27         'A TLS/SSL connection established with [this method] may
28          understand the SSLv3, TLSv1, TLSv1.1 and TLSv1.2 protocols.'
29         """
30 
31         def __init__(self, method=SSL.SSLv23_METHOD, *args, **kwargs):
32             super(ScrapyClientContextFactory, self).__init__(*args, **kwargs)
33             self._ssl_method = method
34 
35         def getCertificateOptions(self):
36             # setting verify=True will require you to provide CAs
37             # to verify against; in other words: it's not that simple
38 
39             # backward-compatible SSL/TLS method:
40             #
41             # * this will respect `method` attribute in often recommended
42             #   `ScrapyClientContextFactory` subclass
43             #   (https://github.com/scrapy/scrapy/issues/1429#issuecomment-131782133)
44             #
45             # * getattr() for `_ssl_method` attribute for context factories
46             #   not calling super(..., self).__init__
47             return CertificateOptions(verify=False,
48                         method=getattr(self, 'method',
49                                        getattr(self, '_ssl_method', None)))
50 
51         # kept for old-style HTTP/1.0 downloader context twisted calls,
52         # e.g. connectSSL()
53         def getContext(self, hostname=None, port=None):
54             return self.getCertificateOptions().getContext()
55 
56         def creatorForNetloc(self, hostname, port):
57             return ScrapyClientTLSOptions(hostname.decode("ascii"), self.getContext())
58 
59 
60     @implementer(IPolicyForHTTPS)
61     class BrowserLikeContextFactory(ScrapyClientContextFactory):
62         """
63         Twisted-recommended context factory for web clients.
64 
65         Quoting http://twistedmatrix.com/documents/current/api/twisted.web.client.Agent.html:
66         "The default is to use a BrowserLikePolicyForHTTPS,
67         so unless you have special requirements you can leave this as-is."
68 
69         creatorForNetloc() is the same as BrowserLikePolicyForHTTPS
70         except this context factory allows setting the TLS/SSL method to use.
71 
72         Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD)
73         which allows TLS protocol negotiation.
74         """
75         def creatorForNetloc(self, hostname, port):
76 
77             # trustRoot set to platformTrust() will use the platform's root CAs.
78             #
79             # This means that a website like https://www.cacert.org will be rejected
80             # by default, since CAcert.org CA certificate is seldom shipped.
81             return optionsForClientTLS(hostname.decode("ascii"),
82                                        trustRoot=platformTrust(),
83                                        extraCertificateOptions={
84                                             'method': self._ssl_method,
85                                        })
86 
87 except ImportError:
88 
89     class ScrapyClientContextFactory(ClientContextFactory):
90         "A SSL context factory which is more permissive against SSL bugs."
91         # see https://github.com/scrapy/scrapy/issues/82
92         # and https://github.com/scrapy/scrapy/issues/26
93         # and https://github.com/scrapy/scrapy/issues/981
94 
95         def __init__(self, method=SSL.SSLv23_METHOD):
96             self.method = method
97 
98         def getContext(self, hostname=None, port=None):
99             ctx = ClientContextFactory.getContext(self)
100             # Enable all workarounds to SSL bugs as documented by
101             # http://www.openssl.org/docs/ssl/SSL_CTX_set_options.html
102             ctx.set_options(SSL.OP_ALL)
103             return ctx
104
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py
--- a/scrapy/core/downloader/contextfactory.py
+++ b/scrapy/core/downloader/contextfactory.py
@@ -13,7 +13,7 @@
     from twisted.web.client import BrowserLikePolicyForHTTPS
     from twisted.web.iweb import IPolicyForHTTPS
 
-    from scrapy.core.downloader.tls import ScrapyClientTLSOptions
+    from scrapy.core.downloader.tls import ScrapyClientTLSOptions, DEFAULT_CIPHERS
 
 
     @implementer(IPolicyForHTTPS)
@@ -46,7 +46,9 @@
             #   not calling super(..., self).__init__
             return CertificateOptions(verify=False,
                         method=getattr(self, 'method',
-                                       getattr(self, '_ssl_method', None)))
+                                       getattr(self, '_ssl_method', None)),
+                        fixBrokenPeers=True,
+                        acceptableCiphers=DEFAULT_CIPHERS)
 
         # kept for old-style HTTP/1.0 downloader context twisted calls,
         # e.g. connectSSL()
diff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py
--- a/scrapy/core/downloader/tls.py
+++ b/scrapy/core/downloader/tls.py
@@ -28,6 +28,7 @@
         SSL_CB_HANDSHAKE_START = 0x10
         SSL_CB_HANDSHAKE_DONE = 0x20
 
+    from twisted.internet.ssl import AcceptableCiphers
     from twisted.internet._sslverify import (ClientTLSOptions,
                                              _maybeSetHostNameIndication,
                                              verifyHostname,
@@ -60,6 +61,8 @@
                         'from host "{}" (exception: {})'.format(
                             self._hostnameASCII, repr(e)))
 
+    DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT')
+
 except ImportError:
     # ImportError should not matter for older Twisted versions
     # as the above is not used in the fallback ScrapyClientContextFactory

{"golden_diff": "diff --git a/scrapy/core/downloader/contextfactory.py b/scrapy/core/downloader/contextfactory.py\n--- a/scrapy/core/downloader/contextfactory.py\n+++ b/scrapy/core/downloader/contextfactory.py\n@@ -13,7 +13,7 @@\n     from twisted.web.client import BrowserLikePolicyForHTTPS\n     from twisted.web.iweb import IPolicyForHTTPS\n \n-    from scrapy.core.downloader.tls import ScrapyClientTLSOptions\n+    from scrapy.core.downloader.tls import ScrapyClientTLSOptions, DEFAULT_CIPHERS\n \n \n     @implementer(IPolicyForHTTPS)\n@@ -46,7 +46,9 @@\n             #   not calling super(..., self).__init__\n             return CertificateOptions(verify=False,\n                         method=getattr(self, 'method',\n-                                       getattr(self, '_ssl_method', None)))\n+                                       getattr(self, '_ssl_method', None)),\n+                        fixBrokenPeers=True,\n+                        acceptableCiphers=DEFAULT_CIPHERS)\n \n         # kept for old-style HTTP/1.0 downloader context twisted calls,\n         # e.g. connectSSL()\ndiff --git a/scrapy/core/downloader/tls.py b/scrapy/core/downloader/tls.py\n--- a/scrapy/core/downloader/tls.py\n+++ b/scrapy/core/downloader/tls.py\n@@ -28,6 +28,7 @@\n         SSL_CB_HANDSHAKE_START = 0x10\n         SSL_CB_HANDSHAKE_DONE = 0x20\n \n+    from twisted.internet.ssl import AcceptableCiphers\n     from twisted.internet._sslverify import (ClientTLSOptions,\n                                              _maybeSetHostNameIndication,\n                                              verifyHostname,\n@@ -60,6 +61,8 @@\n                         'from host \"{}\" (exception: {})'.format(\n                             self._hostnameASCII, repr(e)))\n \n+    DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT')\n+\n except ImportError:\n     # ImportError should not matter for older Twisted versions\n     # as the above is not used in the fallback ScrapyClientContextFactory\n", "issue": "SSL error\nI have lot of errors with SSL websites.\nFor exemple, when I call : `scrapy shell https://subscribe.wsj.com/printpack/`\n\nI have this error : \n\n```\n2016-10-06 22:15:40 [scrapy] DEBUG: Telnet console listening on 127.0.0.1:6043\n2016-10-06 22:15:40 [scrapy] INFO: Spider opened\n2016-10-06 22:15:40 [scrapy] DEBUG: Retrying <GET https://subscribe.wsj.com/printpack/> (failed 1 times): [<twisted.python.failure.Failure OpenSSL.SSL.Error: [('SSL routines', 'SSL23_GET_SERVER_HELLO', 'sslv3 alert handshake failure')]>]\n2016-10-06 22:15:40 [scrapy] DEBUG: Retrying <GET https://subscribe.wsj.com/printpack/> (failed 2 times): [<twisted.python.failure.Failure OpenSSL.SSL.Error: [('SSL routines', 'SSL23_GET_SERVER_HELLO', 'sslv3 alert handshake failure')]>]\n2016-10-06 22:15:40 [scrapy] DEBUG: Gave up retrying <GET https://subscribe.wsj.com/printpack/> (failed 3 times): [<twisted.python.failure.Failure OpenSSL.SSL.Error: [('SSL routines', 'SSL23_GET_SERVER_HELLO', 'sslv3 alert handshake failure')]>]\nTraceback (most recent call last):\n  File \"/usr/local/bin/scrapy\", line 11, in <module>\n    sys.exit(execute())\n  File \"/usr/local/lib/python2.7/dist-packages/scrapy/cmdline.py\", line 142, in execute\n    _run_print_help(parser, _run_command, cmd, args, opts)\n  File \"/usr/local/lib/python2.7/dist-packages/scrapy/cmdline.py\", line 88, in _run_print_help\n    func(*a, **kw)\n  File \"/usr/local/lib/python2.7/dist-packages/scrapy/cmdline.py\", line 149, in _run_command\n    cmd.run(args, opts)\n  File \"/usr/local/lib/python2.7/dist-packages/scrapy/commands/shell.py\", line 71, in run\n    shell.start(url=url)\n  File \"/usr/local/lib/python2.7/dist-packages/scrapy/shell.py\", line 47, in start\n    self.fetch(url, spider)\n  File \"/usr/local/lib/python2.7/dist-packages/scrapy/shell.py\", line 112, in fetch\n    reactor, self._schedule, request, spider)\n  File \"/usr/local/lib/python2.7/dist-packages/twisted/internet/threads.py\", line 122, in blockingCallFromThread\n    result.raiseException()\n  File \"<string>\", line 2, in raiseException\ntwisted.web._newclient.ResponseNeverReceived: [<twisted.python.failure.Failure OpenSSL.SSL.Error: [('SSL routines', 'SSL23_GET_SERVER_HELLO', 'sslv3 alert handshake failure')]>]\n```\n\nHow fix that ?\n\n", "before_files": [{"content": "import logging\nfrom OpenSSL import SSL\n\n\nlogger = logging.getLogger(__name__)\n\nMETHOD_SSLv3 = 'SSLv3'\nMETHOD_TLS = 'TLS'\nMETHOD_TLSv10 = 'TLSv1.0'\nMETHOD_TLSv11 = 'TLSv1.1'\nMETHOD_TLSv12 = 'TLSv1.2'\n\nopenssl_methods = {\n    METHOD_TLS:    SSL.SSLv23_METHOD,                   # protocol negotiation (recommended)\n    METHOD_SSLv3:  SSL.SSLv3_METHOD,                    # SSL 3 (NOT recommended)\n    METHOD_TLSv10: SSL.TLSv1_METHOD,                    # TLS 1.0 only\n    METHOD_TLSv11: getattr(SSL, 'TLSv1_1_METHOD', 5),   # TLS 1.1 only\n    METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6),   # TLS 1.2 only\n}\n\n# ClientTLSOptions requires a recent-enough version of Twisted\ntry:\n\n    # taken from twisted/twisted/internet/_sslverify.py\n    try:\n        from OpenSSL.SSL import SSL_CB_HANDSHAKE_DONE, SSL_CB_HANDSHAKE_START\n    except ImportError:\n        SSL_CB_HANDSHAKE_START = 0x10\n        SSL_CB_HANDSHAKE_DONE = 0x20\n\n    from twisted.internet._sslverify import (ClientTLSOptions,\n                                             _maybeSetHostNameIndication,\n                                             verifyHostname,\n                                             VerificationError)\n\n    class ScrapyClientTLSOptions(ClientTLSOptions):\n        \"\"\"\n        SSL Client connection creator ignoring certificate verification errors\n        (for genuinely invalid certificates or bugs in verification code).\n\n        Same as Twisted's private _sslverify.ClientTLSOptions,\n        except that VerificationError and ValueError exceptions are caught,\n        so that the connection is not closed, only logging warnings.\n        \"\"\"\n\n        def _identityVerifyingInfoCallback(self, connection, where, ret):\n            if where & SSL_CB_HANDSHAKE_START:\n                _maybeSetHostNameIndication(connection, self._hostnameBytes)\n            elif where & SSL_CB_HANDSHAKE_DONE:\n                try:\n                    verifyHostname(connection, self._hostnameASCII)\n                except VerificationError as e:\n                    logger.warning(\n                        'Remote certificate is not valid for hostname \"{}\"; {}'.format(\n                            self._hostnameASCII, e))\n\n                except ValueError as e:\n                    logger.warning(\n                        'Ignoring error while verifying certificate '\n                        'from host \"{}\" (exception: {})'.format(\n                            self._hostnameASCII, repr(e)))\n\nexcept ImportError:\n    # ImportError should not matter for older Twisted versions\n    # as the above is not used in the fallback ScrapyClientContextFactory\n    pass\n", "path": "scrapy/core/downloader/tls.py"}, {"content": "from OpenSSL import SSL\nfrom twisted.internet.ssl import ClientContextFactory\n\ntry:\n\n    from zope.interface.declarations import implementer\n\n    # the following should be available from Twisted 14.0.0\n    from twisted.internet.ssl import (optionsForClientTLS,\n                                      CertificateOptions,\n                                      platformTrust)\n\n    from twisted.web.client import BrowserLikePolicyForHTTPS\n    from twisted.web.iweb import IPolicyForHTTPS\n\n    from scrapy.core.downloader.tls import ScrapyClientTLSOptions\n\n\n    @implementer(IPolicyForHTTPS)\n    class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):\n        \"\"\"\n        Non-peer-certificate verifying HTTPS context factory\n\n        Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD)\n        which allows TLS protocol negotiation\n\n        'A TLS/SSL connection established with [this method] may\n         understand the SSLv3, TLSv1, TLSv1.1 and TLSv1.2 protocols.'\n        \"\"\"\n\n        def __init__(self, method=SSL.SSLv23_METHOD, *args, **kwargs):\n            super(ScrapyClientContextFactory, self).__init__(*args, **kwargs)\n            self._ssl_method = method\n\n        def getCertificateOptions(self):\n            # setting verify=True will require you to provide CAs\n            # to verify against; in other words: it's not that simple\n\n            # backward-compatible SSL/TLS method:\n            #\n            # * this will respect `method` attribute in often recommended\n            #   `ScrapyClientContextFactory` subclass\n            #   (https://github.com/scrapy/scrapy/issues/1429#issuecomment-131782133)\n            #\n            # * getattr() for `_ssl_method` attribute for context factories\n            #   not calling super(..., self).__init__\n            return CertificateOptions(verify=False,\n                        method=getattr(self, 'method',\n                                       getattr(self, '_ssl_method', None)))\n\n        # kept for old-style HTTP/1.0 downloader context twisted calls,\n        # e.g. connectSSL()\n        def getContext(self, hostname=None, port=None):\n            return self.getCertificateOptions().getContext()\n\n        def creatorForNetloc(self, hostname, port):\n            return ScrapyClientTLSOptions(hostname.decode(\"ascii\"), self.getContext())\n\n\n    @implementer(IPolicyForHTTPS)\n    class BrowserLikeContextFactory(ScrapyClientContextFactory):\n        \"\"\"\n        Twisted-recommended context factory for web clients.\n\n        Quoting http://twistedmatrix.com/documents/current/api/twisted.web.client.Agent.html:\n        \"The default is to use a BrowserLikePolicyForHTTPS,\n        so unless you have special requirements you can leave this as-is.\"\n\n        creatorForNetloc() is the same as BrowserLikePolicyForHTTPS\n        except this context factory allows setting the TLS/SSL method to use.\n\n        Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD)\n        which allows TLS protocol negotiation.\n        \"\"\"\n        def creatorForNetloc(self, hostname, port):\n\n            # trustRoot set to platformTrust() will use the platform's root CAs.\n            #\n            # This means that a website like https://www.cacert.org will be rejected\n            # by default, since CAcert.org CA certificate is seldom shipped.\n            return optionsForClientTLS(hostname.decode(\"ascii\"),\n                                       trustRoot=platformTrust(),\n                                       extraCertificateOptions={\n                                            'method': self._ssl_method,\n                                       })\n\nexcept ImportError:\n\n    class ScrapyClientContextFactory(ClientContextFactory):\n        \"A SSL context factory which is more permissive against SSL bugs.\"\n        # see https://github.com/scrapy/scrapy/issues/82\n        # and https://github.com/scrapy/scrapy/issues/26\n        # and https://github.com/scrapy/scrapy/issues/981\n\n        def __init__(self, method=SSL.SSLv23_METHOD):\n            self.method = method\n\n        def getContext(self, hostname=None, port=None):\n            ctx = ClientContextFactory.getContext(self)\n            # Enable all workarounds to SSL bugs as documented by\n            # http://www.openssl.org/docs/ssl/SSL_CTX_set_options.html\n            ctx.set_options(SSL.OP_ALL)\n            return ctx\n", "path": "scrapy/core/downloader/contextfactory.py"}], "after_files": [{"content": "import logging\nfrom OpenSSL import SSL\n\n\nlogger = logging.getLogger(__name__)\n\nMETHOD_SSLv3 = 'SSLv3'\nMETHOD_TLS = 'TLS'\nMETHOD_TLSv10 = 'TLSv1.0'\nMETHOD_TLSv11 = 'TLSv1.1'\nMETHOD_TLSv12 = 'TLSv1.2'\n\nopenssl_methods = {\n    METHOD_TLS:    SSL.SSLv23_METHOD,                   # protocol negotiation (recommended)\n    METHOD_SSLv3:  SSL.SSLv3_METHOD,                    # SSL 3 (NOT recommended)\n    METHOD_TLSv10: SSL.TLSv1_METHOD,                    # TLS 1.0 only\n    METHOD_TLSv11: getattr(SSL, 'TLSv1_1_METHOD', 5),   # TLS 1.1 only\n    METHOD_TLSv12: getattr(SSL, 'TLSv1_2_METHOD', 6),   # TLS 1.2 only\n}\n\n# ClientTLSOptions requires a recent-enough version of Twisted\ntry:\n\n    # taken from twisted/twisted/internet/_sslverify.py\n    try:\n        from OpenSSL.SSL import SSL_CB_HANDSHAKE_DONE, SSL_CB_HANDSHAKE_START\n    except ImportError:\n        SSL_CB_HANDSHAKE_START = 0x10\n        SSL_CB_HANDSHAKE_DONE = 0x20\n\n    from twisted.internet.ssl import AcceptableCiphers\n    from twisted.internet._sslverify import (ClientTLSOptions,\n                                             _maybeSetHostNameIndication,\n                                             verifyHostname,\n                                             VerificationError)\n\n    class ScrapyClientTLSOptions(ClientTLSOptions):\n        \"\"\"\n        SSL Client connection creator ignoring certificate verification errors\n        (for genuinely invalid certificates or bugs in verification code).\n\n        Same as Twisted's private _sslverify.ClientTLSOptions,\n        except that VerificationError and ValueError exceptions are caught,\n        so that the connection is not closed, only logging warnings.\n        \"\"\"\n\n        def _identityVerifyingInfoCallback(self, connection, where, ret):\n            if where & SSL_CB_HANDSHAKE_START:\n                _maybeSetHostNameIndication(connection, self._hostnameBytes)\n            elif where & SSL_CB_HANDSHAKE_DONE:\n                try:\n                    verifyHostname(connection, self._hostnameASCII)\n                except VerificationError as e:\n                    logger.warning(\n                        'Remote certificate is not valid for hostname \"{}\"; {}'.format(\n                            self._hostnameASCII, e))\n\n                except ValueError as e:\n                    logger.warning(\n                        'Ignoring error while verifying certificate '\n                        'from host \"{}\" (exception: {})'.format(\n                            self._hostnameASCII, repr(e)))\n\n    DEFAULT_CIPHERS = AcceptableCiphers.fromOpenSSLCipherString('DEFAULT')\n\nexcept ImportError:\n    # ImportError should not matter for older Twisted versions\n    # as the above is not used in the fallback ScrapyClientContextFactory\n    pass\n", "path": "scrapy/core/downloader/tls.py"}, {"content": "from OpenSSL import SSL\nfrom twisted.internet.ssl import ClientContextFactory\n\ntry:\n\n    from zope.interface.declarations import implementer\n\n    # the following should be available from Twisted 14.0.0\n    from twisted.internet.ssl import (optionsForClientTLS,\n                                      CertificateOptions,\n                                      platformTrust)\n\n    from twisted.web.client import BrowserLikePolicyForHTTPS\n    from twisted.web.iweb import IPolicyForHTTPS\n\n    from scrapy.core.downloader.tls import ScrapyClientTLSOptions, DEFAULT_CIPHERS\n\n\n    @implementer(IPolicyForHTTPS)\n    class ScrapyClientContextFactory(BrowserLikePolicyForHTTPS):\n        \"\"\"\n        Non-peer-certificate verifying HTTPS context factory\n\n        Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD)\n        which allows TLS protocol negotiation\n\n        'A TLS/SSL connection established with [this method] may\n         understand the SSLv3, TLSv1, TLSv1.1 and TLSv1.2 protocols.'\n        \"\"\"\n\n        def __init__(self, method=SSL.SSLv23_METHOD, *args, **kwargs):\n            super(ScrapyClientContextFactory, self).__init__(*args, **kwargs)\n            self._ssl_method = method\n\n        def getCertificateOptions(self):\n            # setting verify=True will require you to provide CAs\n            # to verify against; in other words: it's not that simple\n\n            # backward-compatible SSL/TLS method:\n            #\n            # * this will respect `method` attribute in often recommended\n            #   `ScrapyClientContextFactory` subclass\n            #   (https://github.com/scrapy/scrapy/issues/1429#issuecomment-131782133)\n            #\n            # * getattr() for `_ssl_method` attribute for context factories\n            #   not calling super(..., self).__init__\n            return CertificateOptions(verify=False,\n                        method=getattr(self, 'method',\n                                       getattr(self, '_ssl_method', None)),\n                        fixBrokenPeers=True,\n                        acceptableCiphers=DEFAULT_CIPHERS)\n\n        # kept for old-style HTTP/1.0 downloader context twisted calls,\n        # e.g. connectSSL()\n        def getContext(self, hostname=None, port=None):\n            return self.getCertificateOptions().getContext()\n\n        def creatorForNetloc(self, hostname, port):\n            return ScrapyClientTLSOptions(hostname.decode(\"ascii\"), self.getContext())\n\n\n    @implementer(IPolicyForHTTPS)\n    class BrowserLikeContextFactory(ScrapyClientContextFactory):\n        \"\"\"\n        Twisted-recommended context factory for web clients.\n\n        Quoting http://twistedmatrix.com/documents/current/api/twisted.web.client.Agent.html:\n        \"The default is to use a BrowserLikePolicyForHTTPS,\n        so unless you have special requirements you can leave this as-is.\"\n\n        creatorForNetloc() is the same as BrowserLikePolicyForHTTPS\n        except this context factory allows setting the TLS/SSL method to use.\n\n        Default OpenSSL method is TLS_METHOD (also called SSLv23_METHOD)\n        which allows TLS protocol negotiation.\n        \"\"\"\n        def creatorForNetloc(self, hostname, port):\n\n            # trustRoot set to platformTrust() will use the platform's root CAs.\n            #\n            # This means that a website like https://www.cacert.org will be rejected\n            # by default, since CAcert.org CA certificate is seldom shipped.\n            return optionsForClientTLS(hostname.decode(\"ascii\"),\n                                       trustRoot=platformTrust(),\n                                       extraCertificateOptions={\n                                            'method': self._ssl_method,\n                                       })\n\nexcept ImportError:\n\n    class ScrapyClientContextFactory(ClientContextFactory):\n        \"A SSL context factory which is more permissive against SSL bugs.\"\n        # see https://github.com/scrapy/scrapy/issues/82\n        # and https://github.com/scrapy/scrapy/issues/26\n        # and https://github.com/scrapy/scrapy/issues/981\n\n        def __init__(self, method=SSL.SSLv23_METHOD):\n            self.method = method\n\n        def getContext(self, hostname=None, port=None):\n            ctx = ClientContextFactory.getContext(self)\n            # Enable all workarounds to SSL bugs as documented by\n            # http://www.openssl.org/docs/ssl/SSL_CTX_set_options.html\n            ctx.set_options(SSL.OP_ALL)\n            return ctx\n", "path": "scrapy/core/downloader/contextfactory.py"}]}