prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  Improve structure of Scrapy settings documentation
  #515 got me thinking that some Scrapy settings are missed because they aren't documented in the main settings page (`docs/topics/settings.rst`).
  
  One approach I came up for fixing this was to generate a list of all settings (regardless of which file/doc they're documented) but i couldn't find a way in sphinx to generate a list of all entities of a single role (in this case, "setting").
  
  An alternative approach would to adopt the policy/convention of documenting all settings in the `settings.rst` file (perhaps splitting them in one section per component/middleware/etc) and refer to that file from the other docs (for example, donwloader middlewares in `downloader-middleware.rst` would refer to `settings.rst` for their settings documentation).
  
  Currently, we used a mixed (probably half-transitioned) approach where we document settings separately (in the file where their component is documented) and also mix core & component settings in `settings.rst`. This proposal also aims to resolve that inconsistency.
  
  Thoughts?
  
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `docs/_ext/scrapydocs.py`
  Content:
  ```
  1 from docutils.parsers.rst.roles import set_classes
  2 from docutils import nodes
  3 
  4 def setup(app):
  5     app.add_crossref_type(
  6         directivename = "setting",
  7         rolename      = "setting",
  8         indextemplate = "pair: %s; setting",
  9     )
  10     app.add_crossref_type(
  11         directivename = "signal",
  12         rolename      = "signal",
  13         indextemplate = "pair: %s; signal",
  14     )
  15     app.add_crossref_type(
  16         directivename = "command",
  17         rolename      = "command",
  18         indextemplate = "pair: %s; command",
  19     )
  20     app.add_crossref_type(
  21         directivename = "reqmeta",
  22         rolename      = "reqmeta",
  23         indextemplate = "pair: %s; reqmeta",
  24     )
  25     app.add_role('source', source_role)
  26     app.add_role('commit', commit_role)
  27     app.add_role('issue', issue_role)
  28     app.add_role('rev', rev_role)
  29 
  30 def source_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
  31     ref = 'https://github.com/scrapy/scrapy/blob/master/' + text
  32     set_classes(options)
  33     node = nodes.reference(rawtext, text, refuri=ref, **options)
  34     return [node], []
  35 
  36 def issue_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
  37     ref = 'https://github.com/scrapy/scrapy/issues/' + text
  38     set_classes(options)
  39     node = nodes.reference(rawtext, 'issue ' + text, refuri=ref, **options)
  40     return [node], []
  41 
  42 def commit_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
  43     ref = 'https://github.com/scrapy/scrapy/commit/' + text
  44     set_classes(options)
  45     node = nodes.reference(rawtext, 'commit ' + text, refuri=ref, **options)
  46     return [node], []
  47 
  48 def rev_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
  49     ref = 'http://hg.scrapy.org/scrapy/changeset/' + text
  50     set_classes(options)
  51     node = nodes.reference(rawtext, 'r' + text, refuri=ref, **options)
  52     return [node], []
  53
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py
  --- a/docs/_ext/scrapydocs.py
  +++ b/docs/_ext/scrapydocs.py
  @@ -1,5 +1,82 @@
   from docutils.parsers.rst.roles import set_classes
   from docutils import nodes
  +from sphinx.util.compat import Directive
  +from sphinx.util.nodes import make_refnode
  +from operator import itemgetter
  +
  +
  +class settingslist_node(nodes.General, nodes.Element):
  +    pass
  +
  +
  +class SettingsListDirective(Directive):
  +    def run(self):
  +        return [settingslist_node('')]
  +
  +
  +def is_setting_index(node):
  +    if node.tagname == 'index':
  +        # index entries for setting directives look like:
  +        # [(u'pair', u'SETTING_NAME; setting', u'std:setting-SETTING_NAME', '')]
  +        entry_type, info, refid, _ = node['entries'][0]
  +        return entry_type == 'pair' and info.endswith('; setting')
  +    return False
  +
  +
  +def get_setting_target(node):
  +    # target nodes are placed next to the node in the doc tree
  +    return node.parent[node.parent.index(node) + 1]
  +
  +
  +def get_setting_name_and_refid(node):
  +    """Extract setting name from directive index node"""
  +    entry_type, info, refid, _ = node['entries'][0]
  +    return info.replace('; setting', ''), refid
  +
  +
  +def collect_scrapy_settings_refs(app, doctree):
  +    env = app.builder.env
  +
  +    if not hasattr(env, 'scrapy_all_settings'):
  +        env.scrapy_all_settings = []
  +
  +    for node in doctree.traverse(is_setting_index):
  +        targetnode = get_setting_target(node)
  +        assert isinstance(targetnode, nodes.target), "Next node is not a target"
  +
  +        setting_name, refid = get_setting_name_and_refid(node)
  +
  +        env.scrapy_all_settings.append({
  +            'docname': env.docname,
  +            'setting_name': setting_name,
  +            'refid': refid,
  +        })
  +
  +
  +def make_setting_element(setting_data, app, fromdocname):
  +    refnode = make_refnode(app.builder, fromdocname,
  +                           todocname=setting_data['docname'],
  +                           targetid=setting_data['refid'],
  +                           child=nodes.Text(setting_data['setting_name']))
  +    p = nodes.paragraph()
  +    p += refnode
  +
  +    item = nodes.list_item()
  +    item += p
  +    return item
  +
  +
  +def replace_settingslist_nodes(app, doctree, fromdocname):
  +    env = app.builder.env
  +
  +    for node in doctree.traverse(settingslist_node):
  +        settings_list = nodes.bullet_list()
  +        settings_list.extend([make_setting_element(d, app, fromdocname)
  +                              for d in sorted(env.scrapy_all_settings,
  +                                              key=itemgetter('setting_name'))
  +                              if fromdocname != d['docname']])
  +        node.replace_self(settings_list)
  +
   
   def setup(app):
       app.add_crossref_type(
  @@ -27,6 +104,12 @@
       app.add_role('issue', issue_role)
       app.add_role('rev', rev_role)
   
  +    app.add_node(settingslist_node)
  +    app.add_directive('settingslist', SettingsListDirective)
  +
  +    app.connect('doctree-read', collect_scrapy_settings_refs)
  +    app.connect('doctree-resolved', replace_settingslist_nodes)
  +
   def source_role(name, rawtext, text, lineno, inliner, options={}, content=[]):
       ref = 'https://github.com/scrapy/scrapy/blob/master/' + text
       set_classes(options)
  
verification_info: {"golden_diff": "diff --git a/docs/_ext/scrapydocs.py b/docs/_ext/scrapydocs.py\n--- a/docs/_ext/scrapydocs.py\n+++ b/docs/_ext/scrapydocs.py\n@@ -1,5 +1,82 @@\n from docutils.parsers.rst.roles import set_classes\n from docutils import nodes\n+from sphinx.util.compat import Directive\n+from sphinx.util.nodes import make_refnode\n+from operator import itemgetter\n+\n+\n+class settingslist_node(nodes.General, nodes.Element):\n+    pass\n+\n+\n+class SettingsListDirective(Directive):\n+    def run(self):\n+        return [settingslist_node('')]\n+\n+\n+def is_setting_index(node):\n+    if node.tagname == 'index':\n+        # index entries for setting directives look like:\n+        # [(u'pair', u'SETTING_NAME; setting', u'std:setting-SETTING_NAME', '')]\n+        entry_type, info, refid, _ = node['entries'][0]\n+        return entry_type == 'pair' and info.endswith('; setting')\n+    return False\n+\n+\n+def get_setting_target(node):\n+    # target nodes are placed next to the node in the doc tree\n+    return node.parent[node.parent.index(node) + 1]\n+\n+\n+def get_setting_name_and_refid(node):\n+    \"\"\"Extract setting name from directive index node\"\"\"\n+    entry_type, info, refid, _ = node['entries'][0]\n+    return info.replace('; setting', ''), refid\n+\n+\n+def collect_scrapy_settings_refs(app, doctree):\n+    env = app.builder.env\n+\n+    if not hasattr(env, 'scrapy_all_settings'):\n+        env.scrapy_all_settings = []\n+\n+    for node in doctree.traverse(is_setting_index):\n+        targetnode = get_setting_target(node)\n+        assert isinstance(targetnode, nodes.target), \"Next node is not a target\"\n+\n+        setting_name, refid = get_setting_name_and_refid(node)\n+\n+        env.scrapy_all_settings.append({\n+            'docname': env.docname,\n+            'setting_name': setting_name,\n+            'refid': refid,\n+        })\n+\n+\n+def make_setting_element(setting_data, app, fromdocname):\n+    refnode = make_refnode(app.builder, fromdocname,\n+                           todocname=setting_data['docname'],\n+                           targetid=setting_data['refid'],\n+                           child=nodes.Text(setting_data['setting_name']))\n+    p = nodes.paragraph()\n+    p += refnode\n+\n+    item = nodes.list_item()\n+    item += p\n+    return item\n+\n+\n+def replace_settingslist_nodes(app, doctree, fromdocname):\n+    env = app.builder.env\n+\n+    for node in doctree.traverse(settingslist_node):\n+        settings_list = nodes.bullet_list()\n+        settings_list.extend([make_setting_element(d, app, fromdocname)\n+                              for d in sorted(env.scrapy_all_settings,\n+                                              key=itemgetter('setting_name'))\n+                              if fromdocname != d['docname']])\n+        node.replace_self(settings_list)\n+\n \n def setup(app):\n     app.add_crossref_type(\n@@ -27,6 +104,12 @@\n     app.add_role('issue', issue_role)\n     app.add_role('rev', rev_role)\n \n+    app.add_node(settingslist_node)\n+    app.add_directive('settingslist', SettingsListDirective)\n+\n+    app.connect('doctree-read', collect_scrapy_settings_refs)\n+    app.connect('doctree-resolved', replace_settingslist_nodes)\n+\n def source_role(name, rawtext, text, lineno, inliner, options={}, content=[]):\n     ref = 'https://github.com/scrapy/scrapy/blob/master/' + text\n     set_classes(options)\n", "issue": "Improve structure of Scrapy settings documentation\n#515 got me thinking that some Scrapy settings are missed because they aren't documented in the main settings page (`docs/topics/settings.rst`).\n\nOne approach I came up for fixing this was to generate a list of all settings (regardless of which file/doc they're documented) but i couldn't find a way in sphinx to generate a list of all entities of a single role (in this case, \"setting\").\n\nAn alternative approach would to adopt the policy/convention of documenting all settings in the `settings.rst` file (perhaps splitting them in one section per component/middleware/etc) and refer to that file from the other docs (for example, donwloader middlewares in `downloader-middleware.rst` would refer to `settings.rst` for their settings documentation).\n\nCurrently, we used a mixed (probably half-transitioned) approach where we document settings separately (in the file where their component is documented) and also mix core & component settings in `settings.rst`. This proposal also aims to resolve that inconsistency.\n\nThoughts?\n\n", "before_files": [{"content": "from docutils.parsers.rst.roles import set_classes\nfrom docutils import nodes\n\ndef setup(app):\n    app.add_crossref_type(\n        directivename = \"setting\",\n        rolename      = \"setting\",\n        indextemplate = \"pair: %s; setting\",\n    )\n    app.add_crossref_type(\n        directivename = \"signal\",\n        rolename      = \"signal\",\n        indextemplate = \"pair: %s; signal\",\n    )\n    app.add_crossref_type(\n        directivename = \"command\",\n        rolename      = \"command\",\n        indextemplate = \"pair: %s; command\",\n    )\n    app.add_crossref_type(\n        directivename = \"reqmeta\",\n        rolename      = \"reqmeta\",\n        indextemplate = \"pair: %s; reqmeta\",\n    )\n    app.add_role('source', source_role)\n    app.add_role('commit', commit_role)\n    app.add_role('issue', issue_role)\n    app.add_role('rev', rev_role)\n\ndef source_role(name, rawtext, text, lineno, inliner, options={}, content=[]):\n    ref = 'https://github.com/scrapy/scrapy/blob/master/' + text\n    set_classes(options)\n    node = nodes.reference(rawtext, text, refuri=ref, **options)\n    return [node], []\n\ndef issue_role(name, rawtext, text, lineno, inliner, options={}, content=[]):\n    ref = 'https://github.com/scrapy/scrapy/issues/' + text\n    set_classes(options)\n    node = nodes.reference(rawtext, 'issue ' + text, refuri=ref, **options)\n    return [node], []\n\ndef commit_role(name, rawtext, text, lineno, inliner, options={}, content=[]):\n    ref = 'https://github.com/scrapy/scrapy/commit/' + text\n    set_classes(options)\n    node = nodes.reference(rawtext, 'commit ' + text, refuri=ref, **options)\n    return [node], []\n\ndef rev_role(name, rawtext, text, lineno, inliner, options={}, content=[]):\n    ref = 'http://hg.scrapy.org/scrapy/changeset/' + text\n    set_classes(options)\n    node = nodes.reference(rawtext, 'r' + text, refuri=ref, **options)\n    return [node], []\n", "path": "docs/_ext/scrapydocs.py"}], "after_files": [{"content": "from docutils.parsers.rst.roles import set_classes\nfrom docutils import nodes\nfrom sphinx.util.compat import Directive\nfrom sphinx.util.nodes import make_refnode\nfrom operator import itemgetter\n\n\nclass settingslist_node(nodes.General, nodes.Element):\n    pass\n\n\nclass SettingsListDirective(Directive):\n    def run(self):\n        return [settingslist_node('')]\n\n\ndef is_setting_index(node):\n    if node.tagname == 'index':\n        # index entries for setting directives look like:\n        # [(u'pair', u'SETTING_NAME; setting', u'std:setting-SETTING_NAME', '')]\n        entry_type, info, refid, _ = node['entries'][0]\n        return entry_type == 'pair' and info.endswith('; setting')\n    return False\n\n\ndef get_setting_target(node):\n    # target nodes are placed next to the node in the doc tree\n    return node.parent[node.parent.index(node) + 1]\n\n\ndef get_setting_name_and_refid(node):\n    \"\"\"Extract setting name from directive index node\"\"\"\n    entry_type, info, refid, _ = node['entries'][0]\n    return info.replace('; setting', ''), refid\n\n\ndef collect_scrapy_settings_refs(app, doctree):\n    env = app.builder.env\n\n    if not hasattr(env, 'scrapy_all_settings'):\n        env.scrapy_all_settings = []\n\n    for node in doctree.traverse(is_setting_index):\n        targetnode = get_setting_target(node)\n        assert isinstance(targetnode, nodes.target), \"Next node is not a target\"\n\n        setting_name, refid = get_setting_name_and_refid(node)\n\n        env.scrapy_all_settings.append({\n            'docname': env.docname,\n            'setting_name': setting_name,\n            'refid': refid,\n        })\n\n\ndef make_setting_element(setting_data, app, fromdocname):\n    refnode = make_refnode(app.builder, fromdocname,\n                           todocname=setting_data['docname'],\n                           targetid=setting_data['refid'],\n                           child=nodes.Text(setting_data['setting_name']))\n    p = nodes.paragraph()\n    p += refnode\n\n    item = nodes.list_item()\n    item += p\n    return item\n\n\ndef replace_settingslist_nodes(app, doctree, fromdocname):\n    env = app.builder.env\n\n    for node in doctree.traverse(settingslist_node):\n        settings_list = nodes.bullet_list()\n        settings_list.extend([make_setting_element(d, app, fromdocname)\n                              for d in sorted(env.scrapy_all_settings,\n                                              key=itemgetter('setting_name'))\n                              if fromdocname != d['docname']])\n        node.replace_self(settings_list)\n\n\ndef setup(app):\n    app.add_crossref_type(\n        directivename = \"setting\",\n        rolename      = \"setting\",\n        indextemplate = \"pair: %s; setting\",\n    )\n    app.add_crossref_type(\n        directivename = \"signal\",\n        rolename      = \"signal\",\n        indextemplate = \"pair: %s; signal\",\n    )\n    app.add_crossref_type(\n        directivename = \"command\",\n        rolename      = \"command\",\n        indextemplate = \"pair: %s; command\",\n    )\n    app.add_crossref_type(\n        directivename = \"reqmeta\",\n        rolename      = \"reqmeta\",\n        indextemplate = \"pair: %s; reqmeta\",\n    )\n    app.add_role('source', source_role)\n    app.add_role('commit', commit_role)\n    app.add_role('issue', issue_role)\n    app.add_role('rev', rev_role)\n\n    app.add_node(settingslist_node)\n    app.add_directive('settingslist', SettingsListDirective)\n\n    app.connect('doctree-read', collect_scrapy_settings_refs)\n    app.connect('doctree-resolved', replace_settingslist_nodes)\n\ndef source_role(name, rawtext, text, lineno, inliner, options={}, content=[]):\n    ref = 'https://github.com/scrapy/scrapy/blob/master/' + text\n    set_classes(options)\n    node = nodes.reference(rawtext, text, refuri=ref, **options)\n    return [node], []\n\ndef issue_role(name, rawtext, text, lineno, inliner, options={}, content=[]):\n    ref = 'https://github.com/scrapy/scrapy/issues/' + text\n    set_classes(options)\n    node = nodes.reference(rawtext, 'issue ' + text, refuri=ref, **options)\n    return [node], []\n\ndef commit_role(name, rawtext, text, lineno, inliner, options={}, content=[]):\n    ref = 'https://github.com/scrapy/scrapy/commit/' + text\n    set_classes(options)\n    node = nodes.reference(rawtext, 'commit ' + text, refuri=ref, **options)\n    return [node], []\n\ndef rev_role(name, rawtext, text, lineno, inliner, options={}, content=[]):\n    ref = 'http://hg.scrapy.org/scrapy/changeset/' + text\n    set_classes(options)\n    node = nodes.reference(rawtext, 'r' + text, refuri=ref, **options)\n    return [node], []\n", "path": "docs/_ext/scrapydocs.py"}]}