We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
T - Update robots.txt (remove sites)
**Summary**
There are several page types that we have in our robots.txt, which means that Google can't crawl it to see that we don't want those pages to be indexed.


**Steps To Reproduce (STR)**
1. Go to Search Console
2. go to Coverage > Valid with Warnings > Indexed, though blocked by robots.txt
3. Alternatively: https://developer.mozilla.org/robots.txt


**Actual behavior**
Google has a link to https://developer.mozilla.org/en-US/docs/tag/SpeechRecognitionResult (for example), but that URL is blocked in robots.txt, so it can't follow the link to see that it redirects to https://wiki.developer.mozilla.org/en-US/docs/tag/SpeechRecognitionResult which has a `no-index` directive.


**Expected behavior**
All page types that redirect to the wiki should be removed from the robots.txt of the read-only site, since the wiki has a `no-index` directive on all content already.


**Additional context**
The reason to do this is so we can see actually problematic content show up in our search console reports, instead of this noise. Search console only shows up to 1000 pages as problematic, but there are currently more than 10k warnings, so we might be missing large issues.


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `kuma/landing/views.py`
Content:
```
1 from django.conf import settings
2 from django.http import HttpResponse
3 from django.shortcuts import redirect, render
4 from django.views import static
5 from django.views.decorators.cache import never_cache
6 from django.views.generic import RedirectView
7 
8 from kuma.core.decorators import ensure_wiki_domain, shared_cache_control
9 from kuma.core.utils import is_wiki
10 from kuma.feeder.models import Bundle
11 from kuma.feeder.sections import SECTION_HACKS
12 from kuma.search.models import Filter
13 
14 from .utils import favicon_url
15 
16 
17 @shared_cache_control
18 def contribute_json(request):
19     return static.serve(request, "contribute.json", document_root=settings.ROOT)
20 
21 
22 @shared_cache_control
23 def home(request):
24     """Home page."""
25     context = {}
26     # Need for both wiki and react homepage
27     context["updates"] = list(Bundle.objects.recent_entries(SECTION_HACKS.updates)[:5])
28 
29     # The default template name
30     template_name = "landing/react_homepage.html"
31     if is_wiki(request):
32         template_name = "landing/homepage.html"
33         context["default_filters"] = Filter.objects.default_filters()
34     return render(request, template_name, context)
35 
36 
37 @ensure_wiki_domain
38 @never_cache
39 def maintenance_mode(request):
40     if settings.MAINTENANCE_MODE:
41         return render(request, "landing/maintenance-mode.html")
42     else:
43         return redirect("home")
44 
45 
46 @ensure_wiki_domain
47 @shared_cache_control
48 def promote_buttons(request):
49     """Bug 646192: MDN affiliate buttons"""
50     return render(request, "landing/promote_buttons.html")
51 
52 
53 ROBOTS_ALL_ALLOWED_TXT = """\
54 User-agent: *
55 Sitemap: https://wiki.developer.mozilla.org/sitemap.xml
56 
57 Disallow:
58 """
59 
60 ROBOTS_ALLOWED_TXT = """\
61 User-agent: *
62 Sitemap: https://developer.mozilla.org/sitemap.xml
63 
64 Disallow: /admin/
65 Disallow: /api/
66 Disallow: /*/dashboards/*
67 Disallow: /*docs/feeds
68 Disallow: /*docs/templates
69 Disallow: /*docs*Template:
70 Disallow: /*docs/all
71 Disallow: /*docs/tag*
72 Disallow: /*docs/needs-review*
73 Disallow: /*docs/localization-tag*
74 Disallow: /*docs/with-errors
75 Disallow: /*docs/without-parent
76 Disallow: /*docs/top-level
77 Disallow: /*docs/new
78 Disallow: /*docs/get-documents
79 Disallow: /*docs/submit_akismet_spam
80 Disallow: /*docs/load*
81 Disallow: /*docs/Experiment:*
82 Disallow: /*$api
83 Disallow: /*$compare
84 Disallow: /*$revision
85 Disallow: /*$history
86 Disallow: /*$children
87 Disallow: /*$flag
88 Disallow: /*$locales
89 Disallow: /*$toc
90 Disallow: /*$move
91 Disallow: /*$quick-review
92 Disallow: /*$samples
93 Disallow: /*$revert
94 Disallow: /*$repair_breadcrumbs
95 Disallow: /*$delete
96 Disallow: /*$restore
97 Disallow: /*$purge
98 Disallow: /*$subscribe
99 Disallow: /*$subscribe_to_tree
100 Disallow: /*$vote
101 Disallow: /*docs.json
102 Disallow: /*docs/ckeditor_config.js
103 Disallow: /*/files/
104 Disallow: /media
105 Disallow: /*move-requested
106 Disallow: /*preview-wiki-content
107 Disallow: /*profiles*/edit
108 Disallow: /skins
109 Disallow: /*type=feed
110 Disallow: /*users/
111 """ + "\n".join(
112     "Disallow: /{locale}/search".format(locale=locale)
113     for locale in settings.ENABLED_LOCALES
114 )
115 
116 ROBOTS_GO_AWAY_TXT = """\
117 User-Agent: *
118 Disallow: /
119 """
120 
121 
122 @shared_cache_control
123 def robots_txt(request):
124     """Serve robots.txt that allows or forbids robots."""
125     host = request.get_host()
126     if host in settings.ALLOW_ROBOTS_DOMAINS:
127         robots = ""
128     elif host in settings.ALLOW_ROBOTS_WEB_DOMAINS:
129         if host == settings.WIKI_HOST:
130             robots = ROBOTS_ALL_ALLOWED_TXT
131         else:
132             robots = ROBOTS_ALLOWED_TXT
133     else:
134         robots = ROBOTS_GO_AWAY_TXT
135     return HttpResponse(robots, content_type="text/plain")
136 
137 
138 class FaviconRedirect(RedirectView):
139     """Redirect to the favicon in the static img folder (bug 1402497)"""
140 
141     def get_redirect_url(self, *args, **kwargs):
142         return favicon_url()
143
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/kuma/landing/views.py b/kuma/landing/views.py
--- a/kuma/landing/views.py
+++ b/kuma/landing/views.py
@@ -61,52 +61,14 @@
 User-agent: *
 Sitemap: https://developer.mozilla.org/sitemap.xml
 
-Disallow: /admin/
 Disallow: /api/
-Disallow: /*/dashboards/*
-Disallow: /*docs/feeds
-Disallow: /*docs/templates
-Disallow: /*docs*Template:
-Disallow: /*docs/all
-Disallow: /*docs/tag*
-Disallow: /*docs/needs-review*
-Disallow: /*docs/localization-tag*
-Disallow: /*docs/with-errors
-Disallow: /*docs/without-parent
-Disallow: /*docs/top-level
-Disallow: /*docs/new
 Disallow: /*docs/get-documents
-Disallow: /*docs/submit_akismet_spam
-Disallow: /*docs/load*
 Disallow: /*docs/Experiment:*
-Disallow: /*$api
-Disallow: /*$compare
-Disallow: /*$revision
-Disallow: /*$history
 Disallow: /*$children
-Disallow: /*$flag
-Disallow: /*$locales
-Disallow: /*$toc
-Disallow: /*$move
-Disallow: /*$quick-review
-Disallow: /*$samples
-Disallow: /*$revert
-Disallow: /*$repair_breadcrumbs
-Disallow: /*$delete
-Disallow: /*$restore
-Disallow: /*$purge
-Disallow: /*$subscribe
-Disallow: /*$subscribe_to_tree
-Disallow: /*$vote
 Disallow: /*docs.json
-Disallow: /*docs/ckeditor_config.js
 Disallow: /*/files/
 Disallow: /media
-Disallow: /*move-requested
-Disallow: /*preview-wiki-content
 Disallow: /*profiles*/edit
-Disallow: /skins
-Disallow: /*type=feed
 Disallow: /*users/
 """ + "\n".join(
     "Disallow: /{locale}/search".format(locale=locale)

{"golden_diff": "diff --git a/kuma/landing/views.py b/kuma/landing/views.py\n--- a/kuma/landing/views.py\n+++ b/kuma/landing/views.py\n@@ -61,52 +61,14 @@\n User-agent: *\n Sitemap: https://developer.mozilla.org/sitemap.xml\n \n-Disallow: /admin/\n Disallow: /api/\n-Disallow: /*/dashboards/*\n-Disallow: /*docs/feeds\n-Disallow: /*docs/templates\n-Disallow: /*docs*Template:\n-Disallow: /*docs/all\n-Disallow: /*docs/tag*\n-Disallow: /*docs/needs-review*\n-Disallow: /*docs/localization-tag*\n-Disallow: /*docs/with-errors\n-Disallow: /*docs/without-parent\n-Disallow: /*docs/top-level\n-Disallow: /*docs/new\n Disallow: /*docs/get-documents\n-Disallow: /*docs/submit_akismet_spam\n-Disallow: /*docs/load*\n Disallow: /*docs/Experiment:*\n-Disallow: /*$api\n-Disallow: /*$compare\n-Disallow: /*$revision\n-Disallow: /*$history\n Disallow: /*$children\n-Disallow: /*$flag\n-Disallow: /*$locales\n-Disallow: /*$toc\n-Disallow: /*$move\n-Disallow: /*$quick-review\n-Disallow: /*$samples\n-Disallow: /*$revert\n-Disallow: /*$repair_breadcrumbs\n-Disallow: /*$delete\n-Disallow: /*$restore\n-Disallow: /*$purge\n-Disallow: /*$subscribe\n-Disallow: /*$subscribe_to_tree\n-Disallow: /*$vote\n Disallow: /*docs.json\n-Disallow: /*docs/ckeditor_config.js\n Disallow: /*/files/\n Disallow: /media\n-Disallow: /*move-requested\n-Disallow: /*preview-wiki-content\n Disallow: /*profiles*/edit\n-Disallow: /skins\n-Disallow: /*type=feed\n Disallow: /*users/\n \"\"\" + \"\\n\".join(\n     \"Disallow: /{locale}/search\".format(locale=locale)\n", "issue": "T - Update robots.txt (remove sites)\n**Summary**\r\nThere are several page types that we have in our robots.txt, which means that Google can't crawl it to see that we don't want those pages to be indexed.\r\n\r\n\r\n**Steps To Reproduce (STR)**\r\n1. Go to Search Console\r\n2. go to Coverage > Valid with Warnings > Indexed, though blocked by robots.txt\r\n3. Alternatively: https://developer.mozilla.org/robots.txt\r\n\r\n\r\n**Actual behavior**\r\nGoogle has a link to https://developer.mozilla.org/en-US/docs/tag/SpeechRecognitionResult (for example), but that URL is blocked in robots.txt, so it can't follow the link to see that it redirects to https://wiki.developer.mozilla.org/en-US/docs/tag/SpeechRecognitionResult which has a `no-index` directive.\r\n\r\n\r\n**Expected behavior**\r\nAll page types that redirect to the wiki should be removed from the robots.txt of the read-only site, since the wiki has a `no-index` directive on all content already.\r\n\r\n\r\n**Additional context**\r\nThe reason to do this is so we can see actually problematic content show up in our search console reports, instead of this noise. Search console only shows up to 1000 pages as problematic, but there are currently more than 10k warnings, so we might be missing large issues.\r\n\n", "before_files": [{"content": "from django.conf import settings\nfrom django.http import HttpResponse\nfrom django.shortcuts import redirect, render\nfrom django.views import static\nfrom django.views.decorators.cache import never_cache\nfrom django.views.generic import RedirectView\n\nfrom kuma.core.decorators import ensure_wiki_domain, shared_cache_control\nfrom kuma.core.utils import is_wiki\nfrom kuma.feeder.models import Bundle\nfrom kuma.feeder.sections import SECTION_HACKS\nfrom kuma.search.models import Filter\n\nfrom .utils import favicon_url\n\n\n@shared_cache_control\ndef contribute_json(request):\n    return static.serve(request, \"contribute.json\", document_root=settings.ROOT)\n\n\n@shared_cache_control\ndef home(request):\n    \"\"\"Home page.\"\"\"\n    context = {}\n    # Need for both wiki and react homepage\n    context[\"updates\"] = list(Bundle.objects.recent_entries(SECTION_HACKS.updates)[:5])\n\n    # The default template name\n    template_name = \"landing/react_homepage.html\"\n    if is_wiki(request):\n        template_name = \"landing/homepage.html\"\n        context[\"default_filters\"] = Filter.objects.default_filters()\n    return render(request, template_name, context)\n\n\n@ensure_wiki_domain\n@never_cache\ndef maintenance_mode(request):\n    if settings.MAINTENANCE_MODE:\n        return render(request, \"landing/maintenance-mode.html\")\n    else:\n        return redirect(\"home\")\n\n\n@ensure_wiki_domain\n@shared_cache_control\ndef promote_buttons(request):\n    \"\"\"Bug 646192: MDN affiliate buttons\"\"\"\n    return render(request, \"landing/promote_buttons.html\")\n\n\nROBOTS_ALL_ALLOWED_TXT = \"\"\"\\\nUser-agent: *\nSitemap: https://wiki.developer.mozilla.org/sitemap.xml\n\nDisallow:\n\"\"\"\n\nROBOTS_ALLOWED_TXT = \"\"\"\\\nUser-agent: *\nSitemap: https://developer.mozilla.org/sitemap.xml\n\nDisallow: /admin/\nDisallow: /api/\nDisallow: /*/dashboards/*\nDisallow: /*docs/feeds\nDisallow: /*docs/templates\nDisallow: /*docs*Template:\nDisallow: /*docs/all\nDisallow: /*docs/tag*\nDisallow: /*docs/needs-review*\nDisallow: /*docs/localization-tag*\nDisallow: /*docs/with-errors\nDisallow: /*docs/without-parent\nDisallow: /*docs/top-level\nDisallow: /*docs/new\nDisallow: /*docs/get-documents\nDisallow: /*docs/submit_akismet_spam\nDisallow: /*docs/load*\nDisallow: /*docs/Experiment:*\nDisallow: /*$api\nDisallow: /*$compare\nDisallow: /*$revision\nDisallow: /*$history\nDisallow: /*$children\nDisallow: /*$flag\nDisallow: /*$locales\nDisallow: /*$toc\nDisallow: /*$move\nDisallow: /*$quick-review\nDisallow: /*$samples\nDisallow: /*$revert\nDisallow: /*$repair_breadcrumbs\nDisallow: /*$delete\nDisallow: /*$restore\nDisallow: /*$purge\nDisallow: /*$subscribe\nDisallow: /*$subscribe_to_tree\nDisallow: /*$vote\nDisallow: /*docs.json\nDisallow: /*docs/ckeditor_config.js\nDisallow: /*/files/\nDisallow: /media\nDisallow: /*move-requested\nDisallow: /*preview-wiki-content\nDisallow: /*profiles*/edit\nDisallow: /skins\nDisallow: /*type=feed\nDisallow: /*users/\n\"\"\" + \"\\n\".join(\n    \"Disallow: /{locale}/search\".format(locale=locale)\n    for locale in settings.ENABLED_LOCALES\n)\n\nROBOTS_GO_AWAY_TXT = \"\"\"\\\nUser-Agent: *\nDisallow: /\n\"\"\"\n\n\n@shared_cache_control\ndef robots_txt(request):\n    \"\"\"Serve robots.txt that allows or forbids robots.\"\"\"\n    host = request.get_host()\n    if host in settings.ALLOW_ROBOTS_DOMAINS:\n        robots = \"\"\n    elif host in settings.ALLOW_ROBOTS_WEB_DOMAINS:\n        if host == settings.WIKI_HOST:\n            robots = ROBOTS_ALL_ALLOWED_TXT\n        else:\n            robots = ROBOTS_ALLOWED_TXT\n    else:\n        robots = ROBOTS_GO_AWAY_TXT\n    return HttpResponse(robots, content_type=\"text/plain\")\n\n\nclass FaviconRedirect(RedirectView):\n    \"\"\"Redirect to the favicon in the static img folder (bug 1402497)\"\"\"\n\n    def get_redirect_url(self, *args, **kwargs):\n        return favicon_url()\n", "path": "kuma/landing/views.py"}], "after_files": [{"content": "from django.conf import settings\nfrom django.http import HttpResponse\nfrom django.shortcuts import redirect, render\nfrom django.views import static\nfrom django.views.decorators.cache import never_cache\nfrom django.views.generic import RedirectView\n\nfrom kuma.core.decorators import ensure_wiki_domain, shared_cache_control\nfrom kuma.core.utils import is_wiki\nfrom kuma.feeder.models import Bundle\nfrom kuma.feeder.sections import SECTION_HACKS\nfrom kuma.search.models import Filter\n\nfrom .utils import favicon_url\n\n\n@shared_cache_control\ndef contribute_json(request):\n    return static.serve(request, \"contribute.json\", document_root=settings.ROOT)\n\n\n@shared_cache_control\ndef home(request):\n    \"\"\"Home page.\"\"\"\n    context = {}\n    # Need for both wiki and react homepage\n    context[\"updates\"] = list(Bundle.objects.recent_entries(SECTION_HACKS.updates)[:5])\n\n    # The default template name\n    template_name = \"landing/react_homepage.html\"\n    if is_wiki(request):\n        template_name = \"landing/homepage.html\"\n        context[\"default_filters\"] = Filter.objects.default_filters()\n    return render(request, template_name, context)\n\n\n@ensure_wiki_domain\n@never_cache\ndef maintenance_mode(request):\n    if settings.MAINTENANCE_MODE:\n        return render(request, \"landing/maintenance-mode.html\")\n    else:\n        return redirect(\"home\")\n\n\n@ensure_wiki_domain\n@shared_cache_control\ndef promote_buttons(request):\n    \"\"\"Bug 646192: MDN affiliate buttons\"\"\"\n    return render(request, \"landing/promote_buttons.html\")\n\n\nROBOTS_ALL_ALLOWED_TXT = \"\"\"\\\nUser-agent: *\nSitemap: https://wiki.developer.mozilla.org/sitemap.xml\n\nDisallow:\n\"\"\"\n\nROBOTS_ALLOWED_TXT = \"\"\"\\\nUser-agent: *\nSitemap: https://developer.mozilla.org/sitemap.xml\n\nDisallow: /api/\nDisallow: /*docs/get-documents\nDisallow: /*docs/Experiment:*\nDisallow: /*$children\nDisallow: /*docs.json\nDisallow: /*/files/\nDisallow: /media\nDisallow: /*profiles*/edit\nDisallow: /*users/\n\"\"\" + \"\\n\".join(\n    \"Disallow: /{locale}/search\".format(locale=locale)\n    for locale in settings.ENABLED_LOCALES\n)\n\nROBOTS_GO_AWAY_TXT = \"\"\"\\\nUser-Agent: *\nDisallow: /\n\"\"\"\n\n\n@shared_cache_control\ndef robots_txt(request):\n    \"\"\"Serve robots.txt that allows or forbids robots.\"\"\"\n    host = request.get_host()\n    if host in settings.ALLOW_ROBOTS_DOMAINS:\n        robots = \"\"\n    elif host in settings.ALLOW_ROBOTS_WEB_DOMAINS:\n        if host == settings.WIKI_HOST:\n            robots = ROBOTS_ALL_ALLOWED_TXT\n        else:\n            robots = ROBOTS_ALLOWED_TXT\n    else:\n        robots = ROBOTS_GO_AWAY_TXT\n    return HttpResponse(robots, content_type=\"text/plain\")\n\n\nclass FaviconRedirect(RedirectView):\n    \"\"\"Redirect to the favicon in the static img folder (bug 1402497)\"\"\"\n\n    def get_redirect_url(self, *args, **kwargs):\n        return favicon_url()\n", "path": "kuma/landing/views.py"}]}