We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Add brotlicffi support
Currently, brotli compression is supported when using `brotli` or `brotlipy` (deprecated). We should also support it thorugh `brotlicffi`, the new name of `brotlipy`, which performs worse than `brotli` but works on PyPy.

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `scrapy/downloadermiddlewares/httpcompression.py`
Content:
```
1 from __future__ import annotations
2 
3 import warnings
4 from itertools import chain
5 from logging import getLogger
6 from typing import TYPE_CHECKING, List, Optional, Union
7 
8 from scrapy import Request, Spider, signals
9 from scrapy.crawler import Crawler
10 from scrapy.exceptions import IgnoreRequest, NotConfigured
11 from scrapy.http import Response, TextResponse
12 from scrapy.responsetypes import responsetypes
13 from scrapy.statscollectors import StatsCollector
14 from scrapy.utils._compression import (
15     _DecompressionMaxSizeExceeded,
16     _inflate,
17     _unbrotli,
18     _unzstd,
19 )
20 from scrapy.utils.deprecate import ScrapyDeprecationWarning
21 from scrapy.utils.gz import gunzip
22 
23 if TYPE_CHECKING:
24     # typing.Self requires Python 3.11
25     from typing_extensions import Self
26 
27 logger = getLogger(__name__)
28 
29 ACCEPTED_ENCODINGS: List[bytes] = [b"gzip", b"deflate"]
30 
31 try:
32     import brotli  # noqa: F401
33 except ImportError:
34     pass
35 else:
36     ACCEPTED_ENCODINGS.append(b"br")
37 
38 try:
39     import zstandard  # noqa: F401
40 except ImportError:
41     pass
42 else:
43     ACCEPTED_ENCODINGS.append(b"zstd")
44 
45 
46 class HttpCompressionMiddleware:
47     """This middleware allows compressed (gzip, deflate) traffic to be
48     sent/received from web sites"""
49 
50     def __init__(
51         self,
52         stats: Optional[StatsCollector] = None,
53         *,
54         crawler: Optional[Crawler] = None,
55     ):
56         if not crawler:
57             self.stats = stats
58             self._max_size = 1073741824
59             self._warn_size = 33554432
60             return
61         self.stats = crawler.stats
62         self._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE")
63         self._warn_size = crawler.settings.getint("DOWNLOAD_WARNSIZE")
64         crawler.signals.connect(self.open_spider, signals.spider_opened)
65 
66     @classmethod
67     def from_crawler(cls, crawler: Crawler) -> Self:
68         if not crawler.settings.getbool("COMPRESSION_ENABLED"):
69             raise NotConfigured
70         try:
71             return cls(crawler=crawler)
72         except TypeError:
73             warnings.warn(
74                 "HttpCompressionMiddleware subclasses must either modify "
75                 "their '__init__' method to support a 'crawler' parameter or "
76                 "reimplement their 'from_crawler' method.",
77                 ScrapyDeprecationWarning,
78             )
79             mw = cls()
80             mw.stats = crawler.stats
81             mw._max_size = crawler.settings.getint("DOWNLOAD_MAXSIZE")
82             mw._warn_size = crawler.settings.getint("DOWNLOAD_WARNSIZE")
83             crawler.signals.connect(mw.open_spider, signals.spider_opened)
84             return mw
85 
86     def open_spider(self, spider):
87         if hasattr(spider, "download_maxsize"):
88             self._max_size = spider.download_maxsize
89         if hasattr(spider, "download_warnsize"):
90             self._warn_size = spider.download_warnsize
91 
92     def process_request(
93         self, request: Request, spider: Spider
94     ) -> Union[Request, Response, None]:
95         request.headers.setdefault("Accept-Encoding", b", ".join(ACCEPTED_ENCODINGS))
96         return None
97 
98     def process_response(
99         self, request: Request, response: Response, spider: Spider
100     ) -> Union[Request, Response]:
101         if request.method == "HEAD":
102             return response
103         if isinstance(response, Response):
104             content_encoding = response.headers.getlist("Content-Encoding")
105             if content_encoding:
106                 max_size = request.meta.get("download_maxsize", self._max_size)
107                 warn_size = request.meta.get("download_warnsize", self._warn_size)
108                 try:
109                     decoded_body, content_encoding = self._handle_encoding(
110                         response.body, content_encoding, max_size
111                     )
112                 except _DecompressionMaxSizeExceeded:
113                     raise IgnoreRequest(
114                         f"Ignored response {response} because its body "
115                         f"({len(response.body)} B compressed) exceeded "
116                         f"DOWNLOAD_MAXSIZE ({max_size} B) during "
117                         f"decompression."
118                     )
119                 if len(response.body) < warn_size <= len(decoded_body):
120                     logger.warning(
121                         f"{response} body size after decompression "
122                         f"({len(decoded_body)} B) is larger than the "
123                         f"download warning size ({warn_size} B)."
124                     )
125                 response.headers["Content-Encoding"] = content_encoding
126                 if self.stats:
127                     self.stats.inc_value(
128                         "httpcompression/response_bytes",
129                         len(decoded_body),
130                         spider=spider,
131                     )
132                     self.stats.inc_value(
133                         "httpcompression/response_count", spider=spider
134                     )
135                 respcls = responsetypes.from_args(
136                     headers=response.headers, url=response.url, body=decoded_body
137                 )
138                 kwargs = {"cls": respcls, "body": decoded_body}
139                 if issubclass(respcls, TextResponse):
140                     # force recalculating the encoding until we make sure the
141                     # responsetypes guessing is reliable
142                     kwargs["encoding"] = None
143                 response = response.replace(**kwargs)
144                 if not content_encoding:
145                     del response.headers["Content-Encoding"]
146 
147         return response
148 
149     def _handle_encoding(self, body, content_encoding, max_size):
150         to_decode, to_keep = self._split_encodings(content_encoding)
151         for encoding in to_decode:
152             body = self._decode(body, encoding, max_size)
153         return body, to_keep
154 
155     def _split_encodings(self, content_encoding):
156         to_keep = [
157             encoding.strip().lower()
158             for encoding in chain.from_iterable(
159                 encodings.split(b",") for encodings in content_encoding
160             )
161         ]
162         to_decode = []
163         while to_keep:
164             encoding = to_keep.pop()
165             if encoding not in ACCEPTED_ENCODINGS:
166                 to_keep.append(encoding)
167                 return to_decode, to_keep
168             to_decode.append(encoding)
169         return to_decode, to_keep
170 
171     def _decode(self, body: bytes, encoding: bytes, max_size: int) -> bytes:
172         if encoding in {b"gzip", b"x-gzip"}:
173             return gunzip(body, max_size=max_size)
174         if encoding == b"deflate":
175             return _inflate(body, max_size=max_size)
176         if encoding == b"br" and b"br" in ACCEPTED_ENCODINGS:
177             return _unbrotli(body, max_size=max_size)
178         if encoding == b"zstd" and b"zstd" in ACCEPTED_ENCODINGS:
179             return _unzstd(body, max_size=max_size)
180         return body
181
```

Path: `scrapy/utils/_compression.py`
Content:
```
1 import zlib
2 from io import BytesIO
3 from warnings import warn
4 
5 from scrapy.exceptions import ScrapyDeprecationWarning
6 
7 try:
8     import brotli
9 except ImportError:
10     pass
11 else:
12     try:
13         brotli.Decompressor.process
14     except AttributeError:
15         warn(
16             (
17                 "You have brotlipy installed, and Scrapy will use it, but "
18                 "Scrapy support for brotlipy is deprecated and will stop "
19                 "working in a future version of Scrapy. brotlipy itself is "
20                 "deprecated, it has been superseded by brotlicffi (not "
21                 "currently supported by Scrapy). Please, uninstall brotlipy "
22                 "and install brotli instead. brotlipy has the same import "
23                 "name as brotli, so keeping both installed is strongly "
24                 "discouraged."
25             ),
26             ScrapyDeprecationWarning,
27         )
28 
29         def _brotli_decompress(decompressor, data):
30             return decompressor.decompress(data)
31 
32     else:
33 
34         def _brotli_decompress(decompressor, data):
35             return decompressor.process(data)
36 
37 
38 try:
39     import zstandard
40 except ImportError:
41     pass
42 
43 
44 _CHUNK_SIZE = 65536  # 64 KiB
45 
46 
47 class _DecompressionMaxSizeExceeded(ValueError):
48     pass
49 
50 
51 def _inflate(data: bytes, *, max_size: int = 0) -> bytes:
52     decompressor = zlib.decompressobj()
53     raw_decompressor = zlib.decompressobj(wbits=-15)
54     input_stream = BytesIO(data)
55     output_stream = BytesIO()
56     output_chunk = b"."
57     decompressed_size = 0
58     while output_chunk:
59         input_chunk = input_stream.read(_CHUNK_SIZE)
60         try:
61             output_chunk = decompressor.decompress(input_chunk)
62         except zlib.error:
63             if decompressor != raw_decompressor:
64                 # ugly hack to work with raw deflate content that may
65                 # be sent by microsoft servers. For more information, see:
66                 # http://carsten.codimi.de/gzip.yaws/
67                 # http://www.port80software.com/200ok/archive/2005/10/31/868.aspx
68                 # http://www.gzip.org/zlib/zlib_faq.html#faq38
69                 decompressor = raw_decompressor
70                 output_chunk = decompressor.decompress(input_chunk)
71             else:
72                 raise
73         decompressed_size += len(output_chunk)
74         if max_size and decompressed_size > max_size:
75             raise _DecompressionMaxSizeExceeded(
76                 f"The number of bytes decompressed so far "
77                 f"({decompressed_size} B) exceed the specified maximum "
78                 f"({max_size} B)."
79             )
80         output_stream.write(output_chunk)
81     output_stream.seek(0)
82     return output_stream.read()
83 
84 
85 def _unbrotli(data: bytes, *, max_size: int = 0) -> bytes:
86     decompressor = brotli.Decompressor()
87     input_stream = BytesIO(data)
88     output_stream = BytesIO()
89     output_chunk = b"."
90     decompressed_size = 0
91     while output_chunk:
92         input_chunk = input_stream.read(_CHUNK_SIZE)
93         output_chunk = _brotli_decompress(decompressor, input_chunk)
94         decompressed_size += len(output_chunk)
95         if max_size and decompressed_size > max_size:
96             raise _DecompressionMaxSizeExceeded(
97                 f"The number of bytes decompressed so far "
98                 f"({decompressed_size} B) exceed the specified maximum "
99                 f"({max_size} B)."
100             )
101         output_stream.write(output_chunk)
102     output_stream.seek(0)
103     return output_stream.read()
104 
105 
106 def _unzstd(data: bytes, *, max_size: int = 0) -> bytes:
107     decompressor = zstandard.ZstdDecompressor()
108     stream_reader = decompressor.stream_reader(BytesIO(data))
109     output_stream = BytesIO()
110     output_chunk = b"."
111     decompressed_size = 0
112     while output_chunk:
113         output_chunk = stream_reader.read(_CHUNK_SIZE)
114         decompressed_size += len(output_chunk)
115         if max_size and decompressed_size > max_size:
116             raise _DecompressionMaxSizeExceeded(
117                 f"The number of bytes decompressed so far "
118                 f"({decompressed_size} B) exceed the specified maximum "
119                 f"({max_size} B)."
120             )
121         output_stream.write(output_chunk)
122     output_stream.seek(0)
123     return output_stream.read()
124
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py
--- a/scrapy/downloadermiddlewares/httpcompression.py
+++ b/scrapy/downloadermiddlewares/httpcompression.py
@@ -29,7 +29,10 @@
 ACCEPTED_ENCODINGS: List[bytes] = [b"gzip", b"deflate"]
 
 try:
-    import brotli  # noqa: F401
+    try:
+        import brotli  # noqa: F401
+    except ImportError:
+        import brotlicffi  # noqa: F401
 except ImportError:
     pass
 else:
diff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py
--- a/scrapy/utils/_compression.py
+++ b/scrapy/utils/_compression.py
@@ -5,7 +5,10 @@
 from scrapy.exceptions import ScrapyDeprecationWarning
 
 try:
-    import brotli
+    try:
+        import brotli
+    except ImportError:
+        import brotlicffi as brotli
 except ImportError:
     pass
 else:
@@ -17,9 +20,9 @@
                 "You have brotlipy installed, and Scrapy will use it, but "
                 "Scrapy support for brotlipy is deprecated and will stop "
                 "working in a future version of Scrapy. brotlipy itself is "
-                "deprecated, it has been superseded by brotlicffi (not "
-                "currently supported by Scrapy). Please, uninstall brotlipy "
-                "and install brotli instead. brotlipy has the same import "
+                "deprecated, it has been superseded by brotlicffi. "
+                "Please, uninstall brotlipy "
+                "and install brotli or brotlicffi instead. brotlipy has the same import "
                 "name as brotli, so keeping both installed is strongly "
                 "discouraged."
             ),

{"golden_diff": "diff --git a/scrapy/downloadermiddlewares/httpcompression.py b/scrapy/downloadermiddlewares/httpcompression.py\n--- a/scrapy/downloadermiddlewares/httpcompression.py\n+++ b/scrapy/downloadermiddlewares/httpcompression.py\n@@ -29,7 +29,10 @@\n ACCEPTED_ENCODINGS: List[bytes] = [b\"gzip\", b\"deflate\"]\n \n try:\n-    import brotli  # noqa: F401\n+    try:\n+        import brotli  # noqa: F401\n+    except ImportError:\n+        import brotlicffi  # noqa: F401\n except ImportError:\n     pass\n else:\ndiff --git a/scrapy/utils/_compression.py b/scrapy/utils/_compression.py\n--- a/scrapy/utils/_compression.py\n+++ b/scrapy/utils/_compression.py\n@@ -5,7 +5,10 @@\n from scrapy.exceptions import ScrapyDeprecationWarning\n \n try:\n-    import brotli\n+    try:\n+        import brotli\n+    except ImportError:\n+        import brotlicffi as brotli\n except ImportError:\n     pass\n else:\n@@ -17,9 +20,9 @@\n                 \"You have brotlipy installed, and Scrapy will use it, but \"\n                 \"Scrapy support for brotlipy is deprecated and will stop \"\n                 \"working in a future version of Scrapy. brotlipy itself is \"\n-                \"deprecated, it has been superseded by brotlicffi (not \"\n-                \"currently supported by Scrapy). Please, uninstall brotlipy \"\n-                \"and install brotli instead. brotlipy has the same import \"\n+                \"deprecated, it has been superseded by brotlicffi. \"\n+                \"Please, uninstall brotlipy \"\n+                \"and install brotli or brotlicffi instead. brotlipy has the same import \"\n                 \"name as brotli, so keeping both installed is strongly \"\n                 \"discouraged.\"\n             ),\n", "issue": "Add brotlicffi support\nCurrently, brotli compression is supported when using `brotli` or `brotlipy` (deprecated). We should also support it thorugh `brotlicffi`, the new name of `brotlipy`, which performs worse than `brotli` but works on PyPy.\n", "before_files": [{"content": "from __future__ import annotations\n\nimport warnings\nfrom itertools import chain\nfrom logging import getLogger\nfrom typing import TYPE_CHECKING, List, Optional, Union\n\nfrom scrapy import Request, Spider, signals\nfrom scrapy.crawler import Crawler\nfrom scrapy.exceptions import IgnoreRequest, NotConfigured\nfrom scrapy.http import Response, TextResponse\nfrom scrapy.responsetypes import responsetypes\nfrom scrapy.statscollectors import StatsCollector\nfrom scrapy.utils._compression import (\n    _DecompressionMaxSizeExceeded,\n    _inflate,\n    _unbrotli,\n    _unzstd,\n)\nfrom scrapy.utils.deprecate import ScrapyDeprecationWarning\nfrom scrapy.utils.gz import gunzip\n\nif TYPE_CHECKING:\n    # typing.Self requires Python 3.11\n    from typing_extensions import Self\n\nlogger = getLogger(__name__)\n\nACCEPTED_ENCODINGS: List[bytes] = [b\"gzip\", b\"deflate\"]\n\ntry:\n    import brotli  # noqa: F401\nexcept ImportError:\n    pass\nelse:\n    ACCEPTED_ENCODINGS.append(b\"br\")\n\ntry:\n    import zstandard  # noqa: F401\nexcept ImportError:\n    pass\nelse:\n    ACCEPTED_ENCODINGS.append(b\"zstd\")\n\n\nclass HttpCompressionMiddleware:\n    \"\"\"This middleware allows compressed (gzip, deflate) traffic to be\n    sent/received from web sites\"\"\"\n\n    def __init__(\n        self,\n        stats: Optional[StatsCollector] = None,\n        *,\n        crawler: Optional[Crawler] = None,\n    ):\n        if not crawler:\n            self.stats = stats\n            self._max_size = 1073741824\n            self._warn_size = 33554432\n            return\n        self.stats = crawler.stats\n        self._max_size = crawler.settings.getint(\"DOWNLOAD_MAXSIZE\")\n        self._warn_size = crawler.settings.getint(\"DOWNLOAD_WARNSIZE\")\n        crawler.signals.connect(self.open_spider, signals.spider_opened)\n\n    @classmethod\n    def from_crawler(cls, crawler: Crawler) -> Self:\n        if not crawler.settings.getbool(\"COMPRESSION_ENABLED\"):\n            raise NotConfigured\n        try:\n            return cls(crawler=crawler)\n        except TypeError:\n            warnings.warn(\n                \"HttpCompressionMiddleware subclasses must either modify \"\n                \"their '__init__' method to support a 'crawler' parameter or \"\n                \"reimplement their 'from_crawler' method.\",\n                ScrapyDeprecationWarning,\n            )\n            mw = cls()\n            mw.stats = crawler.stats\n            mw._max_size = crawler.settings.getint(\"DOWNLOAD_MAXSIZE\")\n            mw._warn_size = crawler.settings.getint(\"DOWNLOAD_WARNSIZE\")\n            crawler.signals.connect(mw.open_spider, signals.spider_opened)\n            return mw\n\n    def open_spider(self, spider):\n        if hasattr(spider, \"download_maxsize\"):\n            self._max_size = spider.download_maxsize\n        if hasattr(spider, \"download_warnsize\"):\n            self._warn_size = spider.download_warnsize\n\n    def process_request(\n        self, request: Request, spider: Spider\n    ) -> Union[Request, Response, None]:\n        request.headers.setdefault(\"Accept-Encoding\", b\", \".join(ACCEPTED_ENCODINGS))\n        return None\n\n    def process_response(\n        self, request: Request, response: Response, spider: Spider\n    ) -> Union[Request, Response]:\n        if request.method == \"HEAD\":\n            return response\n        if isinstance(response, Response):\n            content_encoding = response.headers.getlist(\"Content-Encoding\")\n            if content_encoding:\n                max_size = request.meta.get(\"download_maxsize\", self._max_size)\n                warn_size = request.meta.get(\"download_warnsize\", self._warn_size)\n                try:\n                    decoded_body, content_encoding = self._handle_encoding(\n                        response.body, content_encoding, max_size\n                    )\n                except _DecompressionMaxSizeExceeded:\n                    raise IgnoreRequest(\n                        f\"Ignored response {response} because its body \"\n                        f\"({len(response.body)} B compressed) exceeded \"\n                        f\"DOWNLOAD_MAXSIZE ({max_size} B) during \"\n                        f\"decompression.\"\n                    )\n                if len(response.body) < warn_size <= len(decoded_body):\n                    logger.warning(\n                        f\"{response} body size after decompression \"\n                        f\"({len(decoded_body)} B) is larger than the \"\n                        f\"download warning size ({warn_size} B).\"\n                    )\n                response.headers[\"Content-Encoding\"] = content_encoding\n                if self.stats:\n                    self.stats.inc_value(\n                        \"httpcompression/response_bytes\",\n                        len(decoded_body),\n                        spider=spider,\n                    )\n                    self.stats.inc_value(\n                        \"httpcompression/response_count\", spider=spider\n                    )\n                respcls = responsetypes.from_args(\n                    headers=response.headers, url=response.url, body=decoded_body\n                )\n                kwargs = {\"cls\": respcls, \"body\": decoded_body}\n                if issubclass(respcls, TextResponse):\n                    # force recalculating the encoding until we make sure the\n                    # responsetypes guessing is reliable\n                    kwargs[\"encoding\"] = None\n                response = response.replace(**kwargs)\n                if not content_encoding:\n                    del response.headers[\"Content-Encoding\"]\n\n        return response\n\n    def _handle_encoding(self, body, content_encoding, max_size):\n        to_decode, to_keep = self._split_encodings(content_encoding)\n        for encoding in to_decode:\n            body = self._decode(body, encoding, max_size)\n        return body, to_keep\n\n    def _split_encodings(self, content_encoding):\n        to_keep = [\n            encoding.strip().lower()\n            for encoding in chain.from_iterable(\n                encodings.split(b\",\") for encodings in content_encoding\n            )\n        ]\n        to_decode = []\n        while to_keep:\n            encoding = to_keep.pop()\n            if encoding not in ACCEPTED_ENCODINGS:\n                to_keep.append(encoding)\n                return to_decode, to_keep\n            to_decode.append(encoding)\n        return to_decode, to_keep\n\n    def _decode(self, body: bytes, encoding: bytes, max_size: int) -> bytes:\n        if encoding in {b\"gzip\", b\"x-gzip\"}:\n            return gunzip(body, max_size=max_size)\n        if encoding == b\"deflate\":\n            return _inflate(body, max_size=max_size)\n        if encoding == b\"br\" and b\"br\" in ACCEPTED_ENCODINGS:\n            return _unbrotli(body, max_size=max_size)\n        if encoding == b\"zstd\" and b\"zstd\" in ACCEPTED_ENCODINGS:\n            return _unzstd(body, max_size=max_size)\n        return body\n", "path": "scrapy/downloadermiddlewares/httpcompression.py"}, {"content": "import zlib\nfrom io import BytesIO\nfrom warnings import warn\n\nfrom scrapy.exceptions import ScrapyDeprecationWarning\n\ntry:\n    import brotli\nexcept ImportError:\n    pass\nelse:\n    try:\n        brotli.Decompressor.process\n    except AttributeError:\n        warn(\n            (\n                \"You have brotlipy installed, and Scrapy will use it, but \"\n                \"Scrapy support for brotlipy is deprecated and will stop \"\n                \"working in a future version of Scrapy. brotlipy itself is \"\n                \"deprecated, it has been superseded by brotlicffi (not \"\n                \"currently supported by Scrapy). Please, uninstall brotlipy \"\n                \"and install brotli instead. brotlipy has the same import \"\n                \"name as brotli, so keeping both installed is strongly \"\n                \"discouraged.\"\n            ),\n            ScrapyDeprecationWarning,\n        )\n\n        def _brotli_decompress(decompressor, data):\n            return decompressor.decompress(data)\n\n    else:\n\n        def _brotli_decompress(decompressor, data):\n            return decompressor.process(data)\n\n\ntry:\n    import zstandard\nexcept ImportError:\n    pass\n\n\n_CHUNK_SIZE = 65536  # 64 KiB\n\n\nclass _DecompressionMaxSizeExceeded(ValueError):\n    pass\n\n\ndef _inflate(data: bytes, *, max_size: int = 0) -> bytes:\n    decompressor = zlib.decompressobj()\n    raw_decompressor = zlib.decompressobj(wbits=-15)\n    input_stream = BytesIO(data)\n    output_stream = BytesIO()\n    output_chunk = b\".\"\n    decompressed_size = 0\n    while output_chunk:\n        input_chunk = input_stream.read(_CHUNK_SIZE)\n        try:\n            output_chunk = decompressor.decompress(input_chunk)\n        except zlib.error:\n            if decompressor != raw_decompressor:\n                # ugly hack to work with raw deflate content that may\n                # be sent by microsoft servers. For more information, see:\n                # http://carsten.codimi.de/gzip.yaws/\n                # http://www.port80software.com/200ok/archive/2005/10/31/868.aspx\n                # http://www.gzip.org/zlib/zlib_faq.html#faq38\n                decompressor = raw_decompressor\n                output_chunk = decompressor.decompress(input_chunk)\n            else:\n                raise\n        decompressed_size += len(output_chunk)\n        if max_size and decompressed_size > max_size:\n            raise _DecompressionMaxSizeExceeded(\n                f\"The number of bytes decompressed so far \"\n                f\"({decompressed_size} B) exceed the specified maximum \"\n                f\"({max_size} B).\"\n            )\n        output_stream.write(output_chunk)\n    output_stream.seek(0)\n    return output_stream.read()\n\n\ndef _unbrotli(data: bytes, *, max_size: int = 0) -> bytes:\n    decompressor = brotli.Decompressor()\n    input_stream = BytesIO(data)\n    output_stream = BytesIO()\n    output_chunk = b\".\"\n    decompressed_size = 0\n    while output_chunk:\n        input_chunk = input_stream.read(_CHUNK_SIZE)\n        output_chunk = _brotli_decompress(decompressor, input_chunk)\n        decompressed_size += len(output_chunk)\n        if max_size and decompressed_size > max_size:\n            raise _DecompressionMaxSizeExceeded(\n                f\"The number of bytes decompressed so far \"\n                f\"({decompressed_size} B) exceed the specified maximum \"\n                f\"({max_size} B).\"\n            )\n        output_stream.write(output_chunk)\n    output_stream.seek(0)\n    return output_stream.read()\n\n\ndef _unzstd(data: bytes, *, max_size: int = 0) -> bytes:\n    decompressor = zstandard.ZstdDecompressor()\n    stream_reader = decompressor.stream_reader(BytesIO(data))\n    output_stream = BytesIO()\n    output_chunk = b\".\"\n    decompressed_size = 0\n    while output_chunk:\n        output_chunk = stream_reader.read(_CHUNK_SIZE)\n        decompressed_size += len(output_chunk)\n        if max_size and decompressed_size > max_size:\n            raise _DecompressionMaxSizeExceeded(\n                f\"The number of bytes decompressed so far \"\n                f\"({decompressed_size} B) exceed the specified maximum \"\n                f\"({max_size} B).\"\n            )\n        output_stream.write(output_chunk)\n    output_stream.seek(0)\n    return output_stream.read()\n", "path": "scrapy/utils/_compression.py"}], "after_files": [{"content": "from __future__ import annotations\n\nimport warnings\nfrom itertools import chain\nfrom logging import getLogger\nfrom typing import TYPE_CHECKING, List, Optional, Union\n\nfrom scrapy import Request, Spider, signals\nfrom scrapy.crawler import Crawler\nfrom scrapy.exceptions import IgnoreRequest, NotConfigured\nfrom scrapy.http import Response, TextResponse\nfrom scrapy.responsetypes import responsetypes\nfrom scrapy.statscollectors import StatsCollector\nfrom scrapy.utils._compression import (\n    _DecompressionMaxSizeExceeded,\n    _inflate,\n    _unbrotli,\n    _unzstd,\n)\nfrom scrapy.utils.deprecate import ScrapyDeprecationWarning\nfrom scrapy.utils.gz import gunzip\n\nif TYPE_CHECKING:\n    # typing.Self requires Python 3.11\n    from typing_extensions import Self\n\nlogger = getLogger(__name__)\n\nACCEPTED_ENCODINGS: List[bytes] = [b\"gzip\", b\"deflate\"]\n\ntry:\n    try:\n        import brotli  # noqa: F401\n    except ImportError:\n        import brotlicffi  # noqa: F401\nexcept ImportError:\n    pass\nelse:\n    ACCEPTED_ENCODINGS.append(b\"br\")\n\ntry:\n    import zstandard  # noqa: F401\nexcept ImportError:\n    pass\nelse:\n    ACCEPTED_ENCODINGS.append(b\"zstd\")\n\n\nclass HttpCompressionMiddleware:\n    \"\"\"This middleware allows compressed (gzip, deflate) traffic to be\n    sent/received from web sites\"\"\"\n\n    def __init__(\n        self,\n        stats: Optional[StatsCollector] = None,\n        *,\n        crawler: Optional[Crawler] = None,\n    ):\n        if not crawler:\n            self.stats = stats\n            self._max_size = 1073741824\n            self._warn_size = 33554432\n            return\n        self.stats = crawler.stats\n        self._max_size = crawler.settings.getint(\"DOWNLOAD_MAXSIZE\")\n        self._warn_size = crawler.settings.getint(\"DOWNLOAD_WARNSIZE\")\n        crawler.signals.connect(self.open_spider, signals.spider_opened)\n\n    @classmethod\n    def from_crawler(cls, crawler: Crawler) -> Self:\n        if not crawler.settings.getbool(\"COMPRESSION_ENABLED\"):\n            raise NotConfigured\n        try:\n            return cls(crawler=crawler)\n        except TypeError:\n            warnings.warn(\n                \"HttpCompressionMiddleware subclasses must either modify \"\n                \"their '__init__' method to support a 'crawler' parameter or \"\n                \"reimplement their 'from_crawler' method.\",\n                ScrapyDeprecationWarning,\n            )\n            mw = cls()\n            mw.stats = crawler.stats\n            mw._max_size = crawler.settings.getint(\"DOWNLOAD_MAXSIZE\")\n            mw._warn_size = crawler.settings.getint(\"DOWNLOAD_WARNSIZE\")\n            crawler.signals.connect(mw.open_spider, signals.spider_opened)\n            return mw\n\n    def open_spider(self, spider):\n        if hasattr(spider, \"download_maxsize\"):\n            self._max_size = spider.download_maxsize\n        if hasattr(spider, \"download_warnsize\"):\n            self._warn_size = spider.download_warnsize\n\n    def process_request(\n        self, request: Request, spider: Spider\n    ) -> Union[Request, Response, None]:\n        request.headers.setdefault(\"Accept-Encoding\", b\", \".join(ACCEPTED_ENCODINGS))\n        return None\n\n    def process_response(\n        self, request: Request, response: Response, spider: Spider\n    ) -> Union[Request, Response]:\n        if request.method == \"HEAD\":\n            return response\n        if isinstance(response, Response):\n            content_encoding = response.headers.getlist(\"Content-Encoding\")\n            if content_encoding:\n                max_size = request.meta.get(\"download_maxsize\", self._max_size)\n                warn_size = request.meta.get(\"download_warnsize\", self._warn_size)\n                try:\n                    decoded_body, content_encoding = self._handle_encoding(\n                        response.body, content_encoding, max_size\n                    )\n                except _DecompressionMaxSizeExceeded:\n                    raise IgnoreRequest(\n                        f\"Ignored response {response} because its body \"\n                        f\"({len(response.body)} B compressed) exceeded \"\n                        f\"DOWNLOAD_MAXSIZE ({max_size} B) during \"\n                        f\"decompression.\"\n                    )\n                if len(response.body) < warn_size <= len(decoded_body):\n                    logger.warning(\n                        f\"{response} body size after decompression \"\n                        f\"({len(decoded_body)} B) is larger than the \"\n                        f\"download warning size ({warn_size} B).\"\n                    )\n                response.headers[\"Content-Encoding\"] = content_encoding\n                if self.stats:\n                    self.stats.inc_value(\n                        \"httpcompression/response_bytes\",\n                        len(decoded_body),\n                        spider=spider,\n                    )\n                    self.stats.inc_value(\n                        \"httpcompression/response_count\", spider=spider\n                    )\n                respcls = responsetypes.from_args(\n                    headers=response.headers, url=response.url, body=decoded_body\n                )\n                kwargs = {\"cls\": respcls, \"body\": decoded_body}\n                if issubclass(respcls, TextResponse):\n                    # force recalculating the encoding until we make sure the\n                    # responsetypes guessing is reliable\n                    kwargs[\"encoding\"] = None\n                response = response.replace(**kwargs)\n                if not content_encoding:\n                    del response.headers[\"Content-Encoding\"]\n\n        return response\n\n    def _handle_encoding(self, body, content_encoding, max_size):\n        to_decode, to_keep = self._split_encodings(content_encoding)\n        for encoding in to_decode:\n            body = self._decode(body, encoding, max_size)\n        return body, to_keep\n\n    def _split_encodings(self, content_encoding):\n        to_keep = [\n            encoding.strip().lower()\n            for encoding in chain.from_iterable(\n                encodings.split(b\",\") for encodings in content_encoding\n            )\n        ]\n        to_decode = []\n        while to_keep:\n            encoding = to_keep.pop()\n            if encoding not in ACCEPTED_ENCODINGS:\n                to_keep.append(encoding)\n                return to_decode, to_keep\n            to_decode.append(encoding)\n        return to_decode, to_keep\n\n    def _decode(self, body: bytes, encoding: bytes, max_size: int) -> bytes:\n        if encoding in {b\"gzip\", b\"x-gzip\"}:\n            return gunzip(body, max_size=max_size)\n        if encoding == b\"deflate\":\n            return _inflate(body, max_size=max_size)\n        if encoding == b\"br\" and b\"br\" in ACCEPTED_ENCODINGS:\n            return _unbrotli(body, max_size=max_size)\n        if encoding == b\"zstd\" and b\"zstd\" in ACCEPTED_ENCODINGS:\n            return _unzstd(body, max_size=max_size)\n        return body\n", "path": "scrapy/downloadermiddlewares/httpcompression.py"}, {"content": "import zlib\nfrom io import BytesIO\nfrom warnings import warn\n\nfrom scrapy.exceptions import ScrapyDeprecationWarning\n\ntry:\n    try:\n        import brotli\n    except ImportError:\n        import brotlicffi as brotli\nexcept ImportError:\n    pass\nelse:\n    try:\n        brotli.Decompressor.process\n    except AttributeError:\n        warn(\n            (\n                \"You have brotlipy installed, and Scrapy will use it, but \"\n                \"Scrapy support for brotlipy is deprecated and will stop \"\n                \"working in a future version of Scrapy. brotlipy itself is \"\n                \"deprecated, it has been superseded by brotlicffi. \"\n                \"Please, uninstall brotlipy \"\n                \"and install brotli or brotlicffi instead. brotlipy has the same import \"\n                \"name as brotli, so keeping both installed is strongly \"\n                \"discouraged.\"\n            ),\n            ScrapyDeprecationWarning,\n        )\n\n        def _brotli_decompress(decompressor, data):\n            return decompressor.decompress(data)\n\n    else:\n\n        def _brotli_decompress(decompressor, data):\n            return decompressor.process(data)\n\n\ntry:\n    import zstandard\nexcept ImportError:\n    pass\n\n\n_CHUNK_SIZE = 65536  # 64 KiB\n\n\nclass _DecompressionMaxSizeExceeded(ValueError):\n    pass\n\n\ndef _inflate(data: bytes, *, max_size: int = 0) -> bytes:\n    decompressor = zlib.decompressobj()\n    raw_decompressor = zlib.decompressobj(wbits=-15)\n    input_stream = BytesIO(data)\n    output_stream = BytesIO()\n    output_chunk = b\".\"\n    decompressed_size = 0\n    while output_chunk:\n        input_chunk = input_stream.read(_CHUNK_SIZE)\n        try:\n            output_chunk = decompressor.decompress(input_chunk)\n        except zlib.error:\n            if decompressor != raw_decompressor:\n                # ugly hack to work with raw deflate content that may\n                # be sent by microsoft servers. For more information, see:\n                # http://carsten.codimi.de/gzip.yaws/\n                # http://www.port80software.com/200ok/archive/2005/10/31/868.aspx\n                # http://www.gzip.org/zlib/zlib_faq.html#faq38\n                decompressor = raw_decompressor\n                output_chunk = decompressor.decompress(input_chunk)\n            else:\n                raise\n        decompressed_size += len(output_chunk)\n        if max_size and decompressed_size > max_size:\n            raise _DecompressionMaxSizeExceeded(\n                f\"The number of bytes decompressed so far \"\n                f\"({decompressed_size} B) exceed the specified maximum \"\n                f\"({max_size} B).\"\n            )\n        output_stream.write(output_chunk)\n    output_stream.seek(0)\n    return output_stream.read()\n\n\ndef _unbrotli(data: bytes, *, max_size: int = 0) -> bytes:\n    decompressor = brotli.Decompressor()\n    input_stream = BytesIO(data)\n    output_stream = BytesIO()\n    output_chunk = b\".\"\n    decompressed_size = 0\n    while output_chunk:\n        input_chunk = input_stream.read(_CHUNK_SIZE)\n        output_chunk = _brotli_decompress(decompressor, input_chunk)\n        decompressed_size += len(output_chunk)\n        if max_size and decompressed_size > max_size:\n            raise _DecompressionMaxSizeExceeded(\n                f\"The number of bytes decompressed so far \"\n                f\"({decompressed_size} B) exceed the specified maximum \"\n                f\"({max_size} B).\"\n            )\n        output_stream.write(output_chunk)\n    output_stream.seek(0)\n    return output_stream.read()\n\n\ndef _unzstd(data: bytes, *, max_size: int = 0) -> bytes:\n    decompressor = zstandard.ZstdDecompressor()\n    stream_reader = decompressor.stream_reader(BytesIO(data))\n    output_stream = BytesIO()\n    output_chunk = b\".\"\n    decompressed_size = 0\n    while output_chunk:\n        output_chunk = stream_reader.read(_CHUNK_SIZE)\n        decompressed_size += len(output_chunk)\n        if max_size and decompressed_size > max_size:\n            raise _DecompressionMaxSizeExceeded(\n                f\"The number of bytes decompressed so far \"\n                f\"({decompressed_size} B) exceed the specified maximum \"\n                f\"({max_size} B).\"\n            )\n        output_stream.write(output_chunk)\n    output_stream.seek(0)\n    return output_stream.read()\n", "path": "scrapy/utils/_compression.py"}]}