prompt: We are currently solving the following issue within our repository. Here is the issue text:
  --- BEGIN ISSUE ---
  T - Remove all non essential URLs from robots.txt
  **Summary**
  In the past we have added pages we didn't want to be indexed to robots.txt, but that means that Google can't crawl them to see that we don't want those pages to be indexed. We should only have pages in robots.txt that we don't want a robot to crawl (possibly due to performance issues).
  
  
  **Steps To Reproduce (STR)**
  1. Go to Search Console
  2. go to Coverage > Valid with Warnings > Indexed, though blocked by robots.txt
  3. Alternatively: https://developer.mozilla.org/robots.txt
  
  
  **Actual behavior**
  Google has a link to https://developer.mozilla.org/users/google/login/?next=/en-US/docs/MDN/About/Promote (for example), but that URL is blocked in robots.txt, so it can't follow the link to see that it redirects to a sign in page.
  
  
  **Expected behavior**
  Disallow: /*users/ should be removed from robots.txt so Google crawler can follow those urls.
  
  
  **Additional context**
  The reason to do this is so we can see actually problematic content show up in our search console reports, instead of this noise. Search console only shows up to 1000 pages as problematic, but there are currently more than 10k warnings, so we might be missing large issues.
  
  
  --- END ISSUE ---
  
  Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
  --- BEGIN FILES ---
  Path: `kuma/landing/views.py`
  Content:
  ```
  1 from django.conf import settings
  2 from django.http import HttpResponse
  3 from django.shortcuts import redirect, render
  4 from django.views import static
  5 from django.views.decorators.cache import never_cache
  6 from django.views.generic import RedirectView
  7 
  8 from kuma.core.decorators import ensure_wiki_domain, shared_cache_control
  9 from kuma.core.utils import is_wiki
  10 from kuma.feeder.models import Bundle
  11 from kuma.feeder.sections import SECTION_HACKS
  12 from kuma.search.models import Filter
  13 
  14 from .utils import favicon_url
  15 
  16 
  17 @shared_cache_control
  18 def contribute_json(request):
  19     return static.serve(request, "contribute.json", document_root=settings.ROOT)
  20 
  21 
  22 @shared_cache_control
  23 def home(request):
  24     """Home page."""
  25     context = {}
  26     # Need for both wiki and react homepage
  27     context["updates"] = list(Bundle.objects.recent_entries(SECTION_HACKS.updates)[:5])
  28 
  29     # The default template name
  30     template_name = "landing/react_homepage.html"
  31     if is_wiki(request):
  32         template_name = "landing/homepage.html"
  33         context["default_filters"] = Filter.objects.default_filters()
  34     return render(request, template_name, context)
  35 
  36 
  37 @ensure_wiki_domain
  38 @never_cache
  39 def maintenance_mode(request):
  40     if settings.MAINTENANCE_MODE:
  41         return render(request, "landing/maintenance-mode.html")
  42     else:
  43         return redirect("home")
  44 
  45 
  46 @ensure_wiki_domain
  47 @shared_cache_control
  48 def promote_buttons(request):
  49     """Bug 646192: MDN affiliate buttons"""
  50     return render(request, "landing/promote_buttons.html")
  51 
  52 
  53 ROBOTS_ALL_ALLOWED_TXT = """\
  54 User-agent: *
  55 Sitemap: https://wiki.developer.mozilla.org/sitemap.xml
  56 
  57 Disallow:
  58 """
  59 
  60 ROBOTS_ALLOWED_TXT = """\
  61 User-agent: *
  62 Sitemap: https://developer.mozilla.org/sitemap.xml
  63 
  64 Disallow: /api/
  65 Disallow: /*docs/get-documents
  66 Disallow: /*docs/Experiment:*
  67 Disallow: /*$children
  68 Disallow: /*docs.json
  69 Disallow: /*/files/
  70 Disallow: /media
  71 Disallow: /*profiles*/edit
  72 Disallow: /*users/
  73 """ + "\n".join(
  74     "Disallow: /{locale}/search".format(locale=locale)
  75     for locale in settings.ENABLED_LOCALES
  76 )
  77 
  78 ROBOTS_GO_AWAY_TXT = """\
  79 User-Agent: *
  80 Disallow: /
  81 """
  82 
  83 
  84 @shared_cache_control
  85 def robots_txt(request):
  86     """Serve robots.txt that allows or forbids robots."""
  87     host = request.get_host()
  88     if host in settings.ALLOW_ROBOTS_DOMAINS:
  89         robots = ""
  90     elif host in settings.ALLOW_ROBOTS_WEB_DOMAINS:
  91         if host == settings.WIKI_HOST:
  92             robots = ROBOTS_ALL_ALLOWED_TXT
  93         else:
  94             robots = ROBOTS_ALLOWED_TXT
  95     else:
  96         robots = ROBOTS_GO_AWAY_TXT
  97     return HttpResponse(robots, content_type="text/plain")
  98 
  99 
  100 class FaviconRedirect(RedirectView):
  101     """Redirect to the favicon in the static img folder (bug 1402497)"""
  102 
  103     def get_redirect_url(self, *args, **kwargs):
  104         return favicon_url()
  105
  ```
  --- END FILES ---
  
  Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.
  
  Here is an example:
  
  ```diff
  diff --git a/examples/server_async.py b/examples/server_async.py
  --- a/examples/server_async.py
  +++ b/examples/server_async.py
  @@ -313,4 +313,4 @@
   
   
   if __name__ == "__main__":
  -    asyncio.run(run_async_server("."), debug=True)
  +    asyncio.run(run_async_server(), debug=True)
  diff --git a/examples/server_sync.py b/examples/server_sync.py
  --- a/examples/server_sync.py
  +++ b/examples/server_sync.py
  @@ -313,5 +313,5 @@
   
   
   if __name__ == "__main__":
  -    server = run_sync_server(".")
  +    server = run_sync_server()
       server.shutdown()
  
  ```
  
golden_diff: diff --git a/kuma/landing/views.py b/kuma/landing/views.py
  --- a/kuma/landing/views.py
  +++ b/kuma/landing/views.py
  @@ -69,7 +69,6 @@
   Disallow: /*/files/
   Disallow: /media
   Disallow: /*profiles*/edit
  -Disallow: /*users/
   """ + "\n".join(
       "Disallow: /{locale}/search".format(locale=locale)
       for locale in settings.ENABLED_LOCALES
  
verification_info: {"golden_diff": "diff --git a/kuma/landing/views.py b/kuma/landing/views.py\n--- a/kuma/landing/views.py\n+++ b/kuma/landing/views.py\n@@ -69,7 +69,6 @@\n Disallow: /*/files/\n Disallow: /media\n Disallow: /*profiles*/edit\n-Disallow: /*users/\n \"\"\" + \"\\n\".join(\n     \"Disallow: /{locale}/search\".format(locale=locale)\n     for locale in settings.ENABLED_LOCALES\n", "issue": "T - Remove all non essential URLs from robots.txt\n**Summary**\r\nIn the past we have added pages we didn't want to be indexed to robots.txt, but that means that Google can't crawl them to see that we don't want those pages to be indexed. We should only have pages in robots.txt that we don't want a robot to crawl (possibly due to performance issues).\r\n\r\n\r\n**Steps To Reproduce (STR)**\r\n1. Go to Search Console\r\n2. go to Coverage > Valid with Warnings > Indexed, though blocked by robots.txt\r\n3. Alternatively: https://developer.mozilla.org/robots.txt\r\n\r\n\r\n**Actual behavior**\r\nGoogle has a link to https://developer.mozilla.org/users/google/login/?next=/en-US/docs/MDN/About/Promote (for example), but that URL is blocked in robots.txt, so it can't follow the link to see that it redirects to a sign in page.\r\n\r\n\r\n**Expected behavior**\r\nDisallow: /*users/ should be removed from robots.txt so Google crawler can follow those urls.\r\n\r\n\r\n**Additional context**\r\nThe reason to do this is so we can see actually problematic content show up in our search console reports, instead of this noise. Search console only shows up to 1000 pages as problematic, but there are currently more than 10k warnings, so we might be missing large issues.\r\n\n", "before_files": [{"content": "from django.conf import settings\nfrom django.http import HttpResponse\nfrom django.shortcuts import redirect, render\nfrom django.views import static\nfrom django.views.decorators.cache import never_cache\nfrom django.views.generic import RedirectView\n\nfrom kuma.core.decorators import ensure_wiki_domain, shared_cache_control\nfrom kuma.core.utils import is_wiki\nfrom kuma.feeder.models import Bundle\nfrom kuma.feeder.sections import SECTION_HACKS\nfrom kuma.search.models import Filter\n\nfrom .utils import favicon_url\n\n\n@shared_cache_control\ndef contribute_json(request):\n    return static.serve(request, \"contribute.json\", document_root=settings.ROOT)\n\n\n@shared_cache_control\ndef home(request):\n    \"\"\"Home page.\"\"\"\n    context = {}\n    # Need for both wiki and react homepage\n    context[\"updates\"] = list(Bundle.objects.recent_entries(SECTION_HACKS.updates)[:5])\n\n    # The default template name\n    template_name = \"landing/react_homepage.html\"\n    if is_wiki(request):\n        template_name = \"landing/homepage.html\"\n        context[\"default_filters\"] = Filter.objects.default_filters()\n    return render(request, template_name, context)\n\n\n@ensure_wiki_domain\n@never_cache\ndef maintenance_mode(request):\n    if settings.MAINTENANCE_MODE:\n        return render(request, \"landing/maintenance-mode.html\")\n    else:\n        return redirect(\"home\")\n\n\n@ensure_wiki_domain\n@shared_cache_control\ndef promote_buttons(request):\n    \"\"\"Bug 646192: MDN affiliate buttons\"\"\"\n    return render(request, \"landing/promote_buttons.html\")\n\n\nROBOTS_ALL_ALLOWED_TXT = \"\"\"\\\nUser-agent: *\nSitemap: https://wiki.developer.mozilla.org/sitemap.xml\n\nDisallow:\n\"\"\"\n\nROBOTS_ALLOWED_TXT = \"\"\"\\\nUser-agent: *\nSitemap: https://developer.mozilla.org/sitemap.xml\n\nDisallow: /api/\nDisallow: /*docs/get-documents\nDisallow: /*docs/Experiment:*\nDisallow: /*$children\nDisallow: /*docs.json\nDisallow: /*/files/\nDisallow: /media\nDisallow: /*profiles*/edit\nDisallow: /*users/\n\"\"\" + \"\\n\".join(\n    \"Disallow: /{locale}/search\".format(locale=locale)\n    for locale in settings.ENABLED_LOCALES\n)\n\nROBOTS_GO_AWAY_TXT = \"\"\"\\\nUser-Agent: *\nDisallow: /\n\"\"\"\n\n\n@shared_cache_control\ndef robots_txt(request):\n    \"\"\"Serve robots.txt that allows or forbids robots.\"\"\"\n    host = request.get_host()\n    if host in settings.ALLOW_ROBOTS_DOMAINS:\n        robots = \"\"\n    elif host in settings.ALLOW_ROBOTS_WEB_DOMAINS:\n        if host == settings.WIKI_HOST:\n            robots = ROBOTS_ALL_ALLOWED_TXT\n        else:\n            robots = ROBOTS_ALLOWED_TXT\n    else:\n        robots = ROBOTS_GO_AWAY_TXT\n    return HttpResponse(robots, content_type=\"text/plain\")\n\n\nclass FaviconRedirect(RedirectView):\n    \"\"\"Redirect to the favicon in the static img folder (bug 1402497)\"\"\"\n\n    def get_redirect_url(self, *args, **kwargs):\n        return favicon_url()\n", "path": "kuma/landing/views.py"}], "after_files": [{"content": "from django.conf import settings\nfrom django.http import HttpResponse\nfrom django.shortcuts import redirect, render\nfrom django.views import static\nfrom django.views.decorators.cache import never_cache\nfrom django.views.generic import RedirectView\n\nfrom kuma.core.decorators import ensure_wiki_domain, shared_cache_control\nfrom kuma.core.utils import is_wiki\nfrom kuma.feeder.models import Bundle\nfrom kuma.feeder.sections import SECTION_HACKS\nfrom kuma.search.models import Filter\n\nfrom .utils import favicon_url\n\n\n@shared_cache_control\ndef contribute_json(request):\n    return static.serve(request, \"contribute.json\", document_root=settings.ROOT)\n\n\n@shared_cache_control\ndef home(request):\n    \"\"\"Home page.\"\"\"\n    context = {}\n    # Need for both wiki and react homepage\n    context[\"updates\"] = list(Bundle.objects.recent_entries(SECTION_HACKS.updates)[:5])\n\n    # The default template name\n    template_name = \"landing/react_homepage.html\"\n    if is_wiki(request):\n        template_name = \"landing/homepage.html\"\n        context[\"default_filters\"] = Filter.objects.default_filters()\n    return render(request, template_name, context)\n\n\n@ensure_wiki_domain\n@never_cache\ndef maintenance_mode(request):\n    if settings.MAINTENANCE_MODE:\n        return render(request, \"landing/maintenance-mode.html\")\n    else:\n        return redirect(\"home\")\n\n\n@ensure_wiki_domain\n@shared_cache_control\ndef promote_buttons(request):\n    \"\"\"Bug 646192: MDN affiliate buttons\"\"\"\n    return render(request, \"landing/promote_buttons.html\")\n\n\nROBOTS_ALL_ALLOWED_TXT = \"\"\"\\\nUser-agent: *\nSitemap: https://wiki.developer.mozilla.org/sitemap.xml\n\nDisallow:\n\"\"\"\n\nROBOTS_ALLOWED_TXT = \"\"\"\\\nUser-agent: *\nSitemap: https://developer.mozilla.org/sitemap.xml\n\nDisallow: /api/\nDisallow: /*docs/get-documents\nDisallow: /*docs/Experiment:*\nDisallow: /*$children\nDisallow: /*docs.json\nDisallow: /*/files/\nDisallow: /media\nDisallow: /*profiles*/edit\n\"\"\" + \"\\n\".join(\n    \"Disallow: /{locale}/search\".format(locale=locale)\n    for locale in settings.ENABLED_LOCALES\n)\n\nROBOTS_GO_AWAY_TXT = \"\"\"\\\nUser-Agent: *\nDisallow: /\n\"\"\"\n\n\n@shared_cache_control\ndef robots_txt(request):\n    \"\"\"Serve robots.txt that allows or forbids robots.\"\"\"\n    host = request.get_host()\n    if host in settings.ALLOW_ROBOTS_DOMAINS:\n        robots = \"\"\n    elif host in settings.ALLOW_ROBOTS_WEB_DOMAINS:\n        if host == settings.WIKI_HOST:\n            robots = ROBOTS_ALL_ALLOWED_TXT\n        else:\n            robots = ROBOTS_ALLOWED_TXT\n    else:\n        robots = ROBOTS_GO_AWAY_TXT\n    return HttpResponse(robots, content_type=\"text/plain\")\n\n\nclass FaviconRedirect(RedirectView):\n    \"\"\"Redirect to the favicon in the static img folder (bug 1402497)\"\"\"\n\n    def get_redirect_url(self, *args, **kwargs):\n        return favicon_url()\n", "path": "kuma/landing/views.py"}]}