prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  T - Update robots.txt (remove sites)
  **Summary**
  There are several page types that we have in our robots.txt, which means that Google can't crawl it to see that we don't want those pages to be indexed.
  
  
  **Steps To Reproduce (STR)**
  1. Go to Search Console
  2. go to Coverage > Valid with Warnings > Indexed, though blocked by robots.txt
  3. Alternatively: https://developer.mozilla.org/robots.txt
  
  
  **Actual behavior**
  Google has a link to https://developer.mozilla.org/en-US/docs/tag/SpeechRecognitionResult (for example), but that URL is blocked in robots.txt, so it can't follow the link to see that it redirects to https://wiki.developer.mozilla.org/en-US/docs/tag/SpeechRecognitionResult which has a `no-index` directive.
  
  
  **Expected behavior**
  All page types that redirect to the wiki should be removed from the robots.txt of the read-only site, since the wiki has a `no-index` directive on all content already.
  
  
  **Additional context**
  The reason to do this is so we can see actually problematic content show up in our search console reports, instead of this noise. Search console only shows up to 1000 pages as problematic, but there are currently more than 10k warnings, so we might be missing large issues.
  
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `kuma/landing/views.py`
  Content:
  ```
  1 from django.conf import settings
  2 from django.http import HttpResponse
  3 from django.shortcuts import redirect, render
  4 from django.views import static
  5 from django.views.decorators.cache import never_cache
  6 from django.views.generic import RedirectView
  7 
  8 from kuma.core.decorators import ensure_wiki_domain, shared_cache_control
  9 from kuma.core.utils import is_wiki
  10 from kuma.feeder.models import Bundle
  11 from kuma.feeder.sections import SECTION_HACKS
  12 from kuma.search.models import Filter
  13 
  14 from .utils import favicon_url
  15 
  16 
  17 @shared_cache_control
  18 def contribute_json(request):
  19     return static.serve(request, "contribute.json", document_root=settings.ROOT)
  20 
  21 
  22 @shared_cache_control
  23 def home(request):
  24     """Home page."""
  25     context = {}
  26     # Need for both wiki and react homepage
  27     context["updates"] = list(Bundle.objects.recent_entries(SECTION_HACKS.updates)[:5])
  28 
  29     # The default template name
  30     template_name = "landing/react_homepage.html"
  31     if is_wiki(request):
  32         template_name = "landing/homepage.html"
  33         context["default_filters"] = Filter.objects.default_filters()
  34     return render(request, template_name, context)
  35 
  36 
  37 @ensure_wiki_domain
  38 @never_cache
  39 def maintenance_mode(request):
  40     if settings.MAINTENANCE_MODE:
  41         return render(request, "landing/maintenance-mode.html")
  42     else:
  43         return redirect("home")
  44 
  45 
  46 @ensure_wiki_domain
  47 @shared_cache_control
  48 def promote_buttons(request):
  49     """Bug 646192: MDN affiliate buttons"""
  50     return render(request, "landing/promote_buttons.html")
  51 
  52 
  53 ROBOTS_ALL_ALLOWED_TXT = """\
  54 User-agent: *
  55 Sitemap: https://wiki.developer.mozilla.org/sitemap.xml
  56 
  57 Disallow:
  58 """
  59 
  60 ROBOTS_ALLOWED_TXT = """\
  61 User-agent: *
  62 Sitemap: https://developer.mozilla.org/sitemap.xml
  63 
  64 Disallow: /admin/
  65 Disallow: /api/
  66 Disallow: /*/dashboards/*
  67 Disallow: /*docs/feeds
  68 Disallow: /*docs/templates
  69 Disallow: /*docs*Template:
  70 Disallow: /*docs/all
  71 Disallow: /*docs/tag*
  72 Disallow: /*docs/needs-review*
  73 Disallow: /*docs/localization-tag*
  74 Disallow: /*docs/with-errors
  75 Disallow: /*docs/without-parent
  76 Disallow: /*docs/top-level
  77 Disallow: /*docs/new
  78 Disallow: /*docs/get-documents
  79 Disallow: /*docs/submit_akismet_spam
  80 Disallow: /*docs/load*
  81 Disallow: /*docs/Experiment:*
  82 Disallow: /*$api
  83 Disallow: /*$compare
  84 Disallow: /*$revision
  85 Disallow: /*$history
  86 Disallow: /*$children
  87 Disallow: /*$flag
  88 Disallow: /*$locales
  89 Disallow: /*$toc
  90 Disallow: /*$move
  91 Disallow: /*$quick-review
  92 Disallow: /*$samples
  93 Disallow: /*$revert
  94 Disallow: /*$repair_breadcrumbs
  95 Disallow: /*$delete
  96 Disallow: /*$restore
  97 Disallow: /*$purge
  98 Disallow: /*$subscribe
  99 Disallow: /*$subscribe_to_tree
  100 Disallow: /*$vote
  101 Disallow: /*docs.json
  102 Disallow: /*docs/ckeditor_config.js
  103 Disallow: /*/files/
  104 Disallow: /media
  105 Disallow: /*move-requested
  106 Disallow: /*preview-wiki-content
  107 Disallow: /*profiles*/edit
  108 Disallow: /skins
  109 Disallow: /*type=feed
  110 Disallow: /*users/
  111 """ + "\n".join(
  112     "Disallow: /{locale}/search".format(locale=locale)
  113     for locale in settings.ENABLED_LOCALES
  114 )
  115 
  116 ROBOTS_GO_AWAY_TXT = """\
  117 User-Agent: *
  118 Disallow: /
  119 """
  120 
  121 
  122 @shared_cache_control
  123 def robots_txt(request):
  124     """Serve robots.txt that allows or forbids robots."""
  125     host = request.get_host()
  126     if host in settings.ALLOW_ROBOTS_DOMAINS:
  127         robots = ""
  128     elif host in settings.ALLOW_ROBOTS_WEB_DOMAINS:
  129         if host == settings.WIKI_HOST:
  130             robots = ROBOTS_ALL_ALLOWED_TXT
  131         else:
  132             robots = ROBOTS_ALLOWED_TXT
  133     else:
  134         robots = ROBOTS_GO_AWAY_TXT
  135     return HttpResponse(robots, content_type="text/plain")
  136 
  137 
  138 class FaviconRedirect(RedirectView):
  139     """Redirect to the favicon in the static img folder (bug 1402497)"""
  140 
  141     def get_redirect_url(self, *args, **kwargs):
  142         return favicon_url()
  143
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/kuma/landing/views.py b/kuma/landing/views.py
  --- a/kuma/landing/views.py
  +++ b/kuma/landing/views.py
  @@ -61,52 +61,14 @@
   User-agent: *
   Sitemap: https://developer.mozilla.org/sitemap.xml
   
  -Disallow: /admin/
   Disallow: /api/
  -Disallow: /*/dashboards/*
  -Disallow: /*docs/feeds
  -Disallow: /*docs/templates
  -Disallow: /*docs*Template:
  -Disallow: /*docs/all
  -Disallow: /*docs/tag*
  -Disallow: /*docs/needs-review*
  -Disallow: /*docs/localization-tag*
  -Disallow: /*docs/with-errors
  -Disallow: /*docs/without-parent
  -Disallow: /*docs/top-level
  -Disallow: /*docs/new
   Disallow: /*docs/get-documents
  -Disallow: /*docs/submit_akismet_spam
  -Disallow: /*docs/load*
   Disallow: /*docs/Experiment:*
  -Disallow: /*$api
  -Disallow: /*$compare
  -Disallow: /*$revision
  -Disallow: /*$history
   Disallow: /*$children
  -Disallow: /*$flag
  -Disallow: /*$locales
  -Disallow: /*$toc
  -Disallow: /*$move
  -Disallow: /*$quick-review
  -Disallow: /*$samples
  -Disallow: /*$revert
  -Disallow: /*$repair_breadcrumbs
  -Disallow: /*$delete
  -Disallow: /*$restore
  -Disallow: /*$purge
  -Disallow: /*$subscribe
  -Disallow: /*$subscribe_to_tree
  -Disallow: /*$vote
   Disallow: /*docs.json
  -Disallow: /*docs/ckeditor_config.js
   Disallow: /*/files/
   Disallow: /media
  -Disallow: /*move-requested
  -Disallow: /*preview-wiki-content
   Disallow: /*profiles*/edit
  -Disallow: /skins
  -Disallow: /*type=feed
   Disallow: /*users/
   """ + "\n".join(
       "Disallow: /{locale}/search".format(locale=locale)
  
verification_info: {"golden_diff": "diff --git a/kuma/landing/views.py b/kuma/landing/views.py\n--- a/kuma/landing/views.py\n+++ b/kuma/landing/views.py\n@@ -61,52 +61,14 @@\n User-agent: *\n Sitemap: https://developer.mozilla.org/sitemap.xml\n \n-Disallow: /admin/\n Disallow: /api/\n-Disallow: /*/dashboards/*\n-Disallow: /*docs/feeds\n-Disallow: /*docs/templates\n-Disallow: /*docs*Template:\n-Disallow: /*docs/all\n-Disallow: /*docs/tag*\n-Disallow: /*docs/needs-review*\n-Disallow: /*docs/localization-tag*\n-Disallow: /*docs/with-errors\n-Disallow: /*docs/without-parent\n-Disallow: /*docs/top-level\n-Disallow: /*docs/new\n Disallow: /*docs/get-documents\n-Disallow: /*docs/submit_akismet_spam\n-Disallow: /*docs/load*\n Disallow: /*docs/Experiment:*\n-Disallow: /*$api\n-Disallow: /*$compare\n-Disallow: /*$revision\n-Disallow: /*$history\n Disallow: /*$children\n-Disallow: /*$flag\n-Disallow: /*$locales\n-Disallow: /*$toc\n-Disallow: /*$move\n-Disallow: /*$quick-review\n-Disallow: /*$samples\n-Disallow: /*$revert\n-Disallow: /*$repair_breadcrumbs\n-Disallow: /*$delete\n-Disallow: /*$restore\n-Disallow: /*$purge\n-Disallow: /*$subscribe\n-Disallow: /*$subscribe_to_tree\n-Disallow: /*$vote\n Disallow: /*docs.json\n-Disallow: /*docs/ckeditor_config.js\n Disallow: /*/files/\n Disallow: /media\n-Disallow: /*move-requested\n-Disallow: /*preview-wiki-content\n Disallow: /*profiles*/edit\n-Disallow: /skins\n-Disallow: /*type=feed\n Disallow: /*users/\n \"\"\" + \"\\n\".join(\n     \"Disallow: /{locale}/search\".format(locale=locale)\n", "issue": "T - Update robots.txt (remove sites)\n**Summary**\r\nThere are several page types that we have in our robots.txt, which means that Google can't crawl it to see that we don't want those pages to be indexed.\r\n\r\n\r\n**Steps To Reproduce (STR)**\r\n1. Go to Search Console\r\n2. go to Coverage > Valid with Warnings > Indexed, though blocked by robots.txt\r\n3. Alternatively: https://developer.mozilla.org/robots.txt\r\n\r\n\r\n**Actual behavior**\r\nGoogle has a link to https://developer.mozilla.org/en-US/docs/tag/SpeechRecognitionResult (for example), but that URL is blocked in robots.txt, so it can't follow the link to see that it redirects to https://wiki.developer.mozilla.org/en-US/docs/tag/SpeechRecognitionResult which has a `no-index` directive.\r\n\r\n\r\n**Expected behavior**\r\nAll page types that redirect to the wiki should be removed from the robots.txt of the read-only site, since the wiki has a `no-index` directive on all content already.\r\n\r\n\r\n**Additional context**\r\nThe reason to do this is so we can see actually problematic content show up in our search console reports, instead of this noise. Search console only shows up to 1000 pages as problematic, but there are currently more than 10k warnings, so we might be missing large issues.\r\n\n", "before_files": [{"content": "from django.conf import settings\nfrom django.http import HttpResponse\nfrom django.shortcuts import redirect, render\nfrom django.views import static\nfrom django.views.decorators.cache import never_cache\nfrom django.views.generic import RedirectView\n\nfrom kuma.core.decorators import ensure_wiki_domain, shared_cache_control\nfrom kuma.core.utils import is_wiki\nfrom kuma.feeder.models import Bundle\nfrom kuma.feeder.sections import SECTION_HACKS\nfrom kuma.search.models import Filter\n\nfrom .utils import favicon_url\n\n\n@shared_cache_control\ndef contribute_json(request):\n    return static.serve(request, \"contribute.json\", document_root=settings.ROOT)\n\n\n@shared_cache_control\ndef home(request):\n    \"\"\"Home page.\"\"\"\n    context = {}\n    # Need for both wiki and react homepage\n    context[\"updates\"] = list(Bundle.objects.recent_entries(SECTION_HACKS.updates)[:5])\n\n    # The default template name\n    template_name = \"landing/react_homepage.html\"\n    if is_wiki(request):\n        template_name = \"landing/homepage.html\"\n        context[\"default_filters\"] = Filter.objects.default_filters()\n    return render(request, template_name, context)\n\n\n@ensure_wiki_domain\n@never_cache\ndef maintenance_mode(request):\n    if settings.MAINTENANCE_MODE:\n        return render(request, \"landing/maintenance-mode.html\")\n    else:\n        return redirect(\"home\")\n\n\n@ensure_wiki_domain\n@shared_cache_control\ndef promote_buttons(request):\n    \"\"\"Bug 646192: MDN affiliate buttons\"\"\"\n    return render(request, \"landing/promote_buttons.html\")\n\n\nROBOTS_ALL_ALLOWED_TXT = \"\"\"\\\nUser-agent: *\nSitemap: https://wiki.developer.mozilla.org/sitemap.xml\n\nDisallow:\n\"\"\"\n\nROBOTS_ALLOWED_TXT = \"\"\"\\\nUser-agent: *\nSitemap: https://developer.mozilla.org/sitemap.xml\n\nDisallow: /admin/\nDisallow: /api/\nDisallow: /*/dashboards/*\nDisallow: /*docs/feeds\nDisallow: /*docs/templates\nDisallow: /*docs*Template:\nDisallow: /*docs/all\nDisallow: /*docs/tag*\nDisallow: /*docs/needs-review*\nDisallow: /*docs/localization-tag*\nDisallow: /*docs/with-errors\nDisallow: /*docs/without-parent\nDisallow: /*docs/top-level\nDisallow: /*docs/new\nDisallow: /*docs/get-documents\nDisallow: /*docs/submit_akismet_spam\nDisallow: /*docs/load*\nDisallow: /*docs/Experiment:*\nDisallow: /*$api\nDisallow: /*$compare\nDisallow: /*$revision\nDisallow: /*$history\nDisallow: /*$children\nDisallow: /*$flag\nDisallow: /*$locales\nDisallow: /*$toc\nDisallow: /*$move\nDisallow: /*$quick-review\nDisallow: /*$samples\nDisallow: /*$revert\nDisallow: /*$repair_breadcrumbs\nDisallow: /*$delete\nDisallow: /*$restore\nDisallow: /*$purge\nDisallow: /*$subscribe\nDisallow: /*$subscribe_to_tree\nDisallow: /*$vote\nDisallow: /*docs.json\nDisallow: /*docs/ckeditor_config.js\nDisallow: /*/files/\nDisallow: /media\nDisallow: /*move-requested\nDisallow: /*preview-wiki-content\nDisallow: /*profiles*/edit\nDisallow: /skins\nDisallow: /*type=feed\nDisallow: /*users/\n\"\"\" + \"\\n\".join(\n    \"Disallow: /{locale}/search\".format(locale=locale)\n    for locale in settings.ENABLED_LOCALES\n)\n\nROBOTS_GO_AWAY_TXT = \"\"\"\\\nUser-Agent: *\nDisallow: /\n\"\"\"\n\n\n@shared_cache_control\ndef robots_txt(request):\n    \"\"\"Serve robots.txt that allows or forbids robots.\"\"\"\n    host = request.get_host()\n    if host in settings.ALLOW_ROBOTS_DOMAINS:\n        robots = \"\"\n    elif host in settings.ALLOW_ROBOTS_WEB_DOMAINS:\n        if host == settings.WIKI_HOST:\n            robots = ROBOTS_ALL_ALLOWED_TXT\n        else:\n            robots = ROBOTS_ALLOWED_TXT\n    else:\n        robots = ROBOTS_GO_AWAY_TXT\n    return HttpResponse(robots, content_type=\"text/plain\")\n\n\nclass FaviconRedirect(RedirectView):\n    \"\"\"Redirect to the favicon in the static img folder (bug 1402497)\"\"\"\n\n    def get_redirect_url(self, *args, **kwargs):\n        return favicon_url()\n", "path": "kuma/landing/views.py"}], "after_files": [{"content": "from django.conf import settings\nfrom django.http import HttpResponse\nfrom django.shortcuts import redirect, render\nfrom django.views import static\nfrom django.views.decorators.cache import never_cache\nfrom django.views.generic import RedirectView\n\nfrom kuma.core.decorators import ensure_wiki_domain, shared_cache_control\nfrom kuma.core.utils import is_wiki\nfrom kuma.feeder.models import Bundle\nfrom kuma.feeder.sections import SECTION_HACKS\nfrom kuma.search.models import Filter\n\nfrom .utils import favicon_url\n\n\n@shared_cache_control\ndef contribute_json(request):\n    return static.serve(request, \"contribute.json\", document_root=settings.ROOT)\n\n\n@shared_cache_control\ndef home(request):\n    \"\"\"Home page.\"\"\"\n    context = {}\n    # Need for both wiki and react homepage\n    context[\"updates\"] = list(Bundle.objects.recent_entries(SECTION_HACKS.updates)[:5])\n\n    # The default template name\n    template_name = \"landing/react_homepage.html\"\n    if is_wiki(request):\n        template_name = \"landing/homepage.html\"\n        context[\"default_filters\"] = Filter.objects.default_filters()\n    return render(request, template_name, context)\n\n\n@ensure_wiki_domain\n@never_cache\ndef maintenance_mode(request):\n    if settings.MAINTENANCE_MODE:\n        return render(request, \"landing/maintenance-mode.html\")\n    else:\n        return redirect(\"home\")\n\n\n@ensure_wiki_domain\n@shared_cache_control\ndef promote_buttons(request):\n    \"\"\"Bug 646192: MDN affiliate buttons\"\"\"\n    return render(request, \"landing/promote_buttons.html\")\n\n\nROBOTS_ALL_ALLOWED_TXT = \"\"\"\\\nUser-agent: *\nSitemap: https://wiki.developer.mozilla.org/sitemap.xml\n\nDisallow:\n\"\"\"\n\nROBOTS_ALLOWED_TXT = \"\"\"\\\nUser-agent: *\nSitemap: https://developer.mozilla.org/sitemap.xml\n\nDisallow: /api/\nDisallow: /*docs/get-documents\nDisallow: /*docs/Experiment:*\nDisallow: /*$children\nDisallow: /*docs.json\nDisallow: /*/files/\nDisallow: /media\nDisallow: /*profiles*/edit\nDisallow: /*users/\n\"\"\" + \"\\n\".join(\n    \"Disallow: /{locale}/search\".format(locale=locale)\n    for locale in settings.ENABLED_LOCALES\n)\n\nROBOTS_GO_AWAY_TXT = \"\"\"\\\nUser-Agent: *\nDisallow: /\n\"\"\"\n\n\n@shared_cache_control\ndef robots_txt(request):\n    \"\"\"Serve robots.txt that allows or forbids robots.\"\"\"\n    host = request.get_host()\n    if host in settings.ALLOW_ROBOTS_DOMAINS:\n        robots = \"\"\n    elif host in settings.ALLOW_ROBOTS_WEB_DOMAINS:\n        if host == settings.WIKI_HOST:\n            robots = ROBOTS_ALL_ALLOWED_TXT\n        else:\n            robots = ROBOTS_ALLOWED_TXT\n    else:\n        robots = ROBOTS_GO_AWAY_TXT\n    return HttpResponse(robots, content_type=\"text/plain\")\n\n\nclass FaviconRedirect(RedirectView):\n    \"\"\"Redirect to the favicon in the static img folder (bug 1402497)\"\"\"\n\n    def get_redirect_url(self, *args, **kwargs):\n        return favicon_url()\n", "path": "kuma/landing/views.py"}]}