prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  Add logging functionality to `memusage` extension
  
  
  ## Summary
  
  To add logging functionality to memusage extension.
  
  ## Motivation
  
  Scrapy jobs with `MEMUSAGE_ENABLED : True` and defined `MEMUSAGE_LIMIT_MB`  (all jobs on scrapy cloud) can be stopped early due to overuse of RAM memory and receive  `memusage_exceeded` outcome.
  
  First thing required to debug RAM memory leaks - is to identify.. pattern of RAM memory usage.
  Is RAM usage continuously increased at higher rates during runtime?
  or Is RAM usage rapidly increased over limit in last several minutes after hours or even days of stable runtime performance?
  Each reason require different approaches to debug RAM memory leaks.
  
  It will be much easier to debug this if  value of `self.get_virtual_size()` will be added to log in  `_check_limit` method of `memusage` extension
  https://github.com/scrapy/scrapy/blob/6ded3cf4cd134b615239babe28bb28c3ff524b05/scrapy/extensions/memusage.py#L77-L89
  
  ## Describe alternatives you've considered
  
  Applying `MEMUSAGE_WARNING_MB` setting to ~80-90% of `MEMUSAGE_LIMIT_MB` - current implementation of `memusage` extension warns only 1 time so it is not enough data for this.
  
  Manually subclass `memusage` extension with similar changes - as well as any other option it will require to reschedule job. It may be not siutable for jobs with several days(and more) total runtime. So from this side it is preferable that it will be applied in scrapy itself and with enabled this loggin by default.
   
  ## Additional context
  
  Similar functionality previously requested here https://github.com/scrapy/scrapy/issues/2173
  
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `scrapy/extensions/memusage.py`
  Content:
  ```
  1 """
  2 MemoryUsage extension
  3 
  4 See documentation in docs/topics/extensions.rst
  5 """
  6 import sys
  7 import socket
  8 import logging
  9 from pprint import pformat
  10 from importlib import import_module
  11 
  12 from twisted.internet import task
  13 
  14 from scrapy import signals
  15 from scrapy.exceptions import NotConfigured
  16 from scrapy.mail import MailSender
  17 from scrapy.utils.engine import get_engine_status
  18 
  19 logger = logging.getLogger(__name__)
  20 
  21 
  22 class MemoryUsage:
  23 
  24     def __init__(self, crawler):
  25         if not crawler.settings.getbool('MEMUSAGE_ENABLED'):
  26             raise NotConfigured
  27         try:
  28             # stdlib's resource module is only available on unix platforms.
  29             self.resource = import_module('resource')
  30         except ImportError:
  31             raise NotConfigured
  32 
  33         self.crawler = crawler
  34         self.warned = False
  35         self.notify_mails = crawler.settings.getlist('MEMUSAGE_NOTIFY_MAIL')
  36         self.limit = crawler.settings.getint('MEMUSAGE_LIMIT_MB') * 1024 * 1024
  37         self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB') * 1024 * 1024
  38         self.check_interval = crawler.settings.getfloat('MEMUSAGE_CHECK_INTERVAL_SECONDS')
  39         self.mail = MailSender.from_settings(crawler.settings)
  40         crawler.signals.connect(self.engine_started, signal=signals.engine_started)
  41         crawler.signals.connect(self.engine_stopped, signal=signals.engine_stopped)
  42 
  43     @classmethod
  44     def from_crawler(cls, crawler):
  45         return cls(crawler)
  46 
  47     def get_virtual_size(self):
  48         size = self.resource.getrusage(self.resource.RUSAGE_SELF).ru_maxrss
  49         if sys.platform != 'darwin':
  50             # on macOS ru_maxrss is in bytes, on Linux it is in KB
  51             size *= 1024
  52         return size
  53 
  54     def engine_started(self):
  55         self.crawler.stats.set_value('memusage/startup', self.get_virtual_size())
  56         self.tasks = []
  57         tsk = task.LoopingCall(self.update)
  58         self.tasks.append(tsk)
  59         tsk.start(self.check_interval, now=True)
  60         if self.limit:
  61             tsk = task.LoopingCall(self._check_limit)
  62             self.tasks.append(tsk)
  63             tsk.start(self.check_interval, now=True)
  64         if self.warning:
  65             tsk = task.LoopingCall(self._check_warning)
  66             self.tasks.append(tsk)
  67             tsk.start(self.check_interval, now=True)
  68 
  69     def engine_stopped(self):
  70         for tsk in self.tasks:
  71             if tsk.running:
  72                 tsk.stop()
  73 
  74     def update(self):
  75         self.crawler.stats.max_value('memusage/max', self.get_virtual_size())
  76 
  77     def _check_limit(self):
  78         if self.get_virtual_size() > self.limit:
  79             self.crawler.stats.set_value('memusage/limit_reached', 1)
  80             mem = self.limit / 1024 / 1024
  81             logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...",
  82                          {'memusage': mem}, extra={'crawler': self.crawler})
  83             if self.notify_mails:
  84                 subj = (
  85                     f"{self.crawler.settings['BOT_NAME']} terminated: "
  86                     f"memory usage exceeded {mem}M at {socket.gethostname()}"
  87                 )
  88                 self._send_report(self.notify_mails, subj)
  89                 self.crawler.stats.set_value('memusage/limit_notified', 1)
  90 
  91             if self.crawler.engine.spider is not None:
  92                 self.crawler.engine.close_spider(self.crawler.engine.spider, 'memusage_exceeded')
  93             else:
  94                 self.crawler.stop()
  95 
  96     def _check_warning(self):
  97         if self.warned:  # warn only once
  98             return
  99         if self.get_virtual_size() > self.warning:
  100             self.crawler.stats.set_value('memusage/warning_reached', 1)
  101             mem = self.warning / 1024 / 1024
  102             logger.warning("Memory usage reached %(memusage)dM",
  103                            {'memusage': mem}, extra={'crawler': self.crawler})
  104             if self.notify_mails:
  105                 subj = (
  106                     f"{self.crawler.settings['BOT_NAME']} warning: "
  107                     f"memory usage reached {mem}M at {socket.gethostname()}"
  108                 )
  109                 self._send_report(self.notify_mails, subj)
  110                 self.crawler.stats.set_value('memusage/warning_notified', 1)
  111             self.warned = True
  112 
  113     def _send_report(self, rcpts, subject):
  114         """send notification mail with some additional useful info"""
  115         stats = self.crawler.stats
  116         s = f"Memory usage at engine startup : {stats.get_value('memusage/startup')/1024/1024}M\r\n"
  117         s += f"Maximum memory usage          : {stats.get_value('memusage/max')/1024/1024}M\r\n"
  118         s += f"Current memory usage          : {self.get_virtual_size()/1024/1024}M\r\n"
  119 
  120         s += "ENGINE STATUS ------------------------------------------------------- \r\n"
  121         s += "\r\n"
  122         s += pformat(get_engine_status(self.crawler.engine))
  123         s += "\r\n"
  124         self.mail.send(rcpts, subject, s)
  125
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py
  --- a/scrapy/extensions/memusage.py
  +++ b/scrapy/extensions/memusage.py
  @@ -75,7 +75,8 @@
           self.crawler.stats.max_value('memusage/max', self.get_virtual_size())
   
       def _check_limit(self):
  -        if self.get_virtual_size() > self.limit:
  +        current_mem_usage = self.get_virtual_size()
  +        if current_mem_usage > self.limit:
               self.crawler.stats.set_value('memusage/limit_reached', 1)
               mem = self.limit / 1024 / 1024
               logger.error("Memory usage exceeded %(memusage)dM. Shutting down Scrapy...",
  @@ -92,6 +93,8 @@
                   self.crawler.engine.close_spider(self.crawler.engine.spider, 'memusage_exceeded')
               else:
                   self.crawler.stop()
  +        else:
  +            logger.info("Current memory usage is %(virtualsize)dM", {'virtualsize': current_mem_usage / 1024 / 1024})
   
       def _check_warning(self):
           if self.warned:  # warn only once
  
verification_info: {"golden_diff": "diff --git a/scrapy/extensions/memusage.py b/scrapy/extensions/memusage.py\n--- a/scrapy/extensions/memusage.py\n+++ b/scrapy/extensions/memusage.py\n@@ -75,7 +75,8 @@\n         self.crawler.stats.max_value('memusage/max', self.get_virtual_size())\n \n     def _check_limit(self):\n-        if self.get_virtual_size() > self.limit:\n+        current_mem_usage = self.get_virtual_size()\n+        if current_mem_usage > self.limit:\n             self.crawler.stats.set_value('memusage/limit_reached', 1)\n             mem = self.limit / 1024 / 1024\n             logger.error(\"Memory usage exceeded %(memusage)dM. Shutting down Scrapy...\",\n@@ -92,6 +93,8 @@\n                 self.crawler.engine.close_spider(self.crawler.engine.spider, 'memusage_exceeded')\n             else:\n                 self.crawler.stop()\n+        else:\n+            logger.info(\"Current memory usage is %(virtualsize)dM\", {'virtualsize': current_mem_usage / 1024 / 1024})\n \n     def _check_warning(self):\n         if self.warned:  # warn only once\n", "issue": "Add logging functionality to `memusage` extension\n\r\n\r\n## Summary\r\n\r\nTo add logging functionality to memusage extension.\r\n\r\n## Motivation\r\n\r\nScrapy jobs with `MEMUSAGE_ENABLED : True` and defined `MEMUSAGE_LIMIT_MB`  (all jobs on scrapy cloud) can be stopped early due to overuse of RAM memory and receive  `memusage_exceeded` outcome.\r\n\r\nFirst thing required to debug RAM memory leaks - is to identify.. pattern of RAM memory usage.\r\nIs RAM usage continuously increased at higher rates during runtime?\r\nor Is RAM usage rapidly increased over limit in last several minutes after hours or even days of stable runtime performance?\r\nEach reason require different approaches to debug RAM memory leaks.\r\n\r\nIt will be much easier to debug this if  value of `self.get_virtual_size()` will be added to log in  `_check_limit` method of `memusage` extension\r\nhttps://github.com/scrapy/scrapy/blob/6ded3cf4cd134b615239babe28bb28c3ff524b05/scrapy/extensions/memusage.py#L77-L89\r\n\r\n## Describe alternatives you've considered\r\n\r\nApplying `MEMUSAGE_WARNING_MB` setting to ~80-90% of `MEMUSAGE_LIMIT_MB` - current implementation of `memusage` extension warns only 1 time so it is not enough data for this.\r\n\r\nManually subclass `memusage` extension with similar changes - as well as any other option it will require to reschedule job. It may be not siutable for jobs with several days(and more) total runtime. So from this side it is preferable that it will be applied in scrapy itself and with enabled this loggin by default.\r\n \r\n## Additional context\r\n\r\nSimilar functionality previously requested here https://github.com/scrapy/scrapy/issues/2173\r\n\n", "before_files": [{"content": "\"\"\"\nMemoryUsage extension\n\nSee documentation in docs/topics/extensions.rst\n\"\"\"\nimport sys\nimport socket\nimport logging\nfrom pprint import pformat\nfrom importlib import import_module\n\nfrom twisted.internet import task\n\nfrom scrapy import signals\nfrom scrapy.exceptions import NotConfigured\nfrom scrapy.mail import MailSender\nfrom scrapy.utils.engine import get_engine_status\n\nlogger = logging.getLogger(__name__)\n\n\nclass MemoryUsage:\n\n    def __init__(self, crawler):\n        if not crawler.settings.getbool('MEMUSAGE_ENABLED'):\n            raise NotConfigured\n        try:\n            # stdlib's resource module is only available on unix platforms.\n            self.resource = import_module('resource')\n        except ImportError:\n            raise NotConfigured\n\n        self.crawler = crawler\n        self.warned = False\n        self.notify_mails = crawler.settings.getlist('MEMUSAGE_NOTIFY_MAIL')\n        self.limit = crawler.settings.getint('MEMUSAGE_LIMIT_MB') * 1024 * 1024\n        self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB') * 1024 * 1024\n        self.check_interval = crawler.settings.getfloat('MEMUSAGE_CHECK_INTERVAL_SECONDS')\n        self.mail = MailSender.from_settings(crawler.settings)\n        crawler.signals.connect(self.engine_started, signal=signals.engine_started)\n        crawler.signals.connect(self.engine_stopped, signal=signals.engine_stopped)\n\n    @classmethod\n    def from_crawler(cls, crawler):\n        return cls(crawler)\n\n    def get_virtual_size(self):\n        size = self.resource.getrusage(self.resource.RUSAGE_SELF).ru_maxrss\n        if sys.platform != 'darwin':\n            # on macOS ru_maxrss is in bytes, on Linux it is in KB\n            size *= 1024\n        return size\n\n    def engine_started(self):\n        self.crawler.stats.set_value('memusage/startup', self.get_virtual_size())\n        self.tasks = []\n        tsk = task.LoopingCall(self.update)\n        self.tasks.append(tsk)\n        tsk.start(self.check_interval, now=True)\n        if self.limit:\n            tsk = task.LoopingCall(self._check_limit)\n            self.tasks.append(tsk)\n            tsk.start(self.check_interval, now=True)\n        if self.warning:\n            tsk = task.LoopingCall(self._check_warning)\n            self.tasks.append(tsk)\n            tsk.start(self.check_interval, now=True)\n\n    def engine_stopped(self):\n        for tsk in self.tasks:\n            if tsk.running:\n                tsk.stop()\n\n    def update(self):\n        self.crawler.stats.max_value('memusage/max', self.get_virtual_size())\n\n    def _check_limit(self):\n        if self.get_virtual_size() > self.limit:\n            self.crawler.stats.set_value('memusage/limit_reached', 1)\n            mem = self.limit / 1024 / 1024\n            logger.error(\"Memory usage exceeded %(memusage)dM. Shutting down Scrapy...\",\n                         {'memusage': mem}, extra={'crawler': self.crawler})\n            if self.notify_mails:\n                subj = (\n                    f\"{self.crawler.settings['BOT_NAME']} terminated: \"\n                    f\"memory usage exceeded {mem}M at {socket.gethostname()}\"\n                )\n                self._send_report(self.notify_mails, subj)\n                self.crawler.stats.set_value('memusage/limit_notified', 1)\n\n            if self.crawler.engine.spider is not None:\n                self.crawler.engine.close_spider(self.crawler.engine.spider, 'memusage_exceeded')\n            else:\n                self.crawler.stop()\n\n    def _check_warning(self):\n        if self.warned:  # warn only once\n            return\n        if self.get_virtual_size() > self.warning:\n            self.crawler.stats.set_value('memusage/warning_reached', 1)\n            mem = self.warning / 1024 / 1024\n            logger.warning(\"Memory usage reached %(memusage)dM\",\n                           {'memusage': mem}, extra={'crawler': self.crawler})\n            if self.notify_mails:\n                subj = (\n                    f\"{self.crawler.settings['BOT_NAME']} warning: \"\n                    f\"memory usage reached {mem}M at {socket.gethostname()}\"\n                )\n                self._send_report(self.notify_mails, subj)\n                self.crawler.stats.set_value('memusage/warning_notified', 1)\n            self.warned = True\n\n    def _send_report(self, rcpts, subject):\n        \"\"\"send notification mail with some additional useful info\"\"\"\n        stats = self.crawler.stats\n        s = f\"Memory usage at engine startup : {stats.get_value('memusage/startup')/1024/1024}M\\r\\n\"\n        s += f\"Maximum memory usage          : {stats.get_value('memusage/max')/1024/1024}M\\r\\n\"\n        s += f\"Current memory usage          : {self.get_virtual_size()/1024/1024}M\\r\\n\"\n\n        s += \"ENGINE STATUS ------------------------------------------------------- \\r\\n\"\n        s += \"\\r\\n\"\n        s += pformat(get_engine_status(self.crawler.engine))\n        s += \"\\r\\n\"\n        self.mail.send(rcpts, subject, s)\n", "path": "scrapy/extensions/memusage.py"}], "after_files": [{"content": "\"\"\"\nMemoryUsage extension\n\nSee documentation in docs/topics/extensions.rst\n\"\"\"\nimport sys\nimport socket\nimport logging\nfrom pprint import pformat\nfrom importlib import import_module\n\nfrom twisted.internet import task\n\nfrom scrapy import signals\nfrom scrapy.exceptions import NotConfigured\nfrom scrapy.mail import MailSender\nfrom scrapy.utils.engine import get_engine_status\n\nlogger = logging.getLogger(__name__)\n\n\nclass MemoryUsage:\n\n    def __init__(self, crawler):\n        if not crawler.settings.getbool('MEMUSAGE_ENABLED'):\n            raise NotConfigured\n        try:\n            # stdlib's resource module is only available on unix platforms.\n            self.resource = import_module('resource')\n        except ImportError:\n            raise NotConfigured\n\n        self.crawler = crawler\n        self.warned = False\n        self.notify_mails = crawler.settings.getlist('MEMUSAGE_NOTIFY_MAIL')\n        self.limit = crawler.settings.getint('MEMUSAGE_LIMIT_MB') * 1024 * 1024\n        self.warning = crawler.settings.getint('MEMUSAGE_WARNING_MB') * 1024 * 1024\n        self.check_interval = crawler.settings.getfloat('MEMUSAGE_CHECK_INTERVAL_SECONDS')\n        self.mail = MailSender.from_settings(crawler.settings)\n        crawler.signals.connect(self.engine_started, signal=signals.engine_started)\n        crawler.signals.connect(self.engine_stopped, signal=signals.engine_stopped)\n\n    @classmethod\n    def from_crawler(cls, crawler):\n        return cls(crawler)\n\n    def get_virtual_size(self):\n        size = self.resource.getrusage(self.resource.RUSAGE_SELF).ru_maxrss\n        if sys.platform != 'darwin':\n            # on macOS ru_maxrss is in bytes, on Linux it is in KB\n            size *= 1024\n        return size\n\n    def engine_started(self):\n        self.crawler.stats.set_value('memusage/startup', self.get_virtual_size())\n        self.tasks = []\n        tsk = task.LoopingCall(self.update)\n        self.tasks.append(tsk)\n        tsk.start(self.check_interval, now=True)\n        if self.limit:\n            tsk = task.LoopingCall(self._check_limit)\n            self.tasks.append(tsk)\n            tsk.start(self.check_interval, now=True)\n        if self.warning:\n            tsk = task.LoopingCall(self._check_warning)\n            self.tasks.append(tsk)\n            tsk.start(self.check_interval, now=True)\n\n    def engine_stopped(self):\n        for tsk in self.tasks:\n            if tsk.running:\n                tsk.stop()\n\n    def update(self):\n        self.crawler.stats.max_value('memusage/max', self.get_virtual_size())\n\n    def _check_limit(self):\n        current_mem_usage = self.get_virtual_size()\n        if current_mem_usage > self.limit:\n            self.crawler.stats.set_value('memusage/limit_reached', 1)\n            mem = self.limit / 1024 / 1024\n            logger.error(\"Memory usage exceeded %(memusage)dM. Shutting down Scrapy...\",\n                         {'memusage': mem}, extra={'crawler': self.crawler})\n            if self.notify_mails:\n                subj = (\n                    f\"{self.crawler.settings['BOT_NAME']} terminated: \"\n                    f\"memory usage exceeded {mem}M at {socket.gethostname()}\"\n                )\n                self._send_report(self.notify_mails, subj)\n                self.crawler.stats.set_value('memusage/limit_notified', 1)\n\n            if self.crawler.engine.spider is not None:\n                self.crawler.engine.close_spider(self.crawler.engine.spider, 'memusage_exceeded')\n            else:\n                self.crawler.stop()\n        else:\n            logger.info(\"Current memory usage is %(virtualsize)dM\", {'virtualsize': current_mem_usage / 1024 / 1024})\n\n    def _check_warning(self):\n        if self.warned:  # warn only once\n            return\n        if self.get_virtual_size() > self.warning:\n            self.crawler.stats.set_value('memusage/warning_reached', 1)\n            mem = self.warning / 1024 / 1024\n            logger.warning(\"Memory usage reached %(memusage)dM\",\n                           {'memusage': mem}, extra={'crawler': self.crawler})\n            if self.notify_mails:\n                subj = (\n                    f\"{self.crawler.settings['BOT_NAME']} warning: \"\n                    f\"memory usage reached {mem}M at {socket.gethostname()}\"\n                )\n                self._send_report(self.notify_mails, subj)\n                self.crawler.stats.set_value('memusage/warning_notified', 1)\n            self.warned = True\n\n    def _send_report(self, rcpts, subject):\n        \"\"\"send notification mail with some additional useful info\"\"\"\n        stats = self.crawler.stats\n        s = f\"Memory usage at engine startup : {stats.get_value('memusage/startup')/1024/1024}M\\r\\n\"\n        s += f\"Maximum memory usage          : {stats.get_value('memusage/max')/1024/1024}M\\r\\n\"\n        s += f\"Current memory usage          : {self.get_virtual_size()/1024/1024}M\\r\\n\"\n\n        s += \"ENGINE STATUS ------------------------------------------------------- \\r\\n\"\n        s += \"\\r\\n\"\n        s += pformat(get_engine_status(self.crawler.engine))\n        s += \"\\r\\n\"\n        self.mail.send(rcpts, subject, s)\n", "path": "scrapy/extensions/memusage.py"}]}