We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Custom robots.txt support?
We've talked about blowing away the protected designation, so not sure if it makes sense to put special case on the protected privacy level, but maybe a separate option for docs that shouldn't be crawled?

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `readthedocs/core/urls/subdomain.py`
Content:
```
1 """URL configurations for subdomains."""
2 from __future__ import absolute_import
3 
4 from functools import reduce
5 from operator import add
6 
7 from django.conf.urls import url
8 from django.conf import settings
9 from django.conf.urls.static import static
10 
11 from readthedocs.core.views.serve import (
12     redirect_page_with_filename,
13     redirect_project_slug, serve_docs
14 )
15 from readthedocs.core.views import (
16     server_error_500,
17     server_error_404,
18 )
19 from readthedocs.constants import pattern_opts
20 
21 handler500 = server_error_500
22 handler404 = server_error_404
23 
24 subdomain_urls = [
25     url(r'^(?:|projects/(?P<subproject_slug>{project_slug})/)'
26         r'page/(?P<filename>.*)$'.format(**pattern_opts),
27         redirect_page_with_filename,
28         name='docs_detail'),
29 
30     url((r'^(?:|projects/(?P<subproject_slug>{project_slug})/)$').format(**pattern_opts),
31         redirect_project_slug,
32         name='redirect_project_slug'),
33 
34     url((r'^(?:|projects/(?P<subproject_slug>{project_slug})/)'
35          r'(?P<lang_slug>{lang_slug})/'
36          r'(?P<version_slug>{version_slug})/'
37          r'(?P<filename>{filename_slug})$'.format(**pattern_opts)),
38         serve_docs,
39         name='docs_detail'),
40 ]
41 
42 groups = [subdomain_urls]
43 
44 # Needed to serve media locally
45 if getattr(settings, 'DEBUG', False):
46     groups.insert(0, static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT))
47 
48 urlpatterns = reduce(add, groups)
49
```

Path: `readthedocs/core/views/serve.py`
Content:
```
1 # -*- coding: utf-8 -*-
2 """
3 Doc serving from Python.
4 
5 In production there are two modes,
6 * Serving from public symlinks in nginx (readthedocs.org & readthedocs.com)
7 * Serving from private symlinks in Python (readthedocs.com only)
8 
9 In development, we have two modes:
10 * Serving from public symlinks in Python
11 * Serving from private symlinks in Python
12 
13 This means we should only serve from public symlinks in dev,
14 and generally default to serving from private symlinks in Python only.
15 
16 Privacy
17 -------
18 
19 These views will take into account the version privacy level.
20 
21 Settings
22 --------
23 
24 PYTHON_MEDIA (False) - Set this to True to serve docs & media from Python
25 SERVE_DOCS (['private']) - The list of ['private', 'public'] docs to serve.
26 """
27 
28 from __future__ import (
29     absolute_import, division, print_function, unicode_literals)
30 
31 import logging
32 import mimetypes
33 import os
34 from functools import wraps
35 
36 from django.conf import settings
37 from django.http import HttpResponse, HttpResponseRedirect, Http404
38 from django.shortcuts import get_object_or_404
39 from django.shortcuts import render
40 from django.utils.encoding import iri_to_uri
41 from django.views.static import serve
42 
43 from readthedocs.builds.models import Version
44 from readthedocs.core.permissions import AdminPermission
45 from readthedocs.core.resolver import resolve, resolve_path
46 from readthedocs.core.symlink import PrivateSymlink, PublicSymlink
47 from readthedocs.projects import constants
48 from readthedocs.projects.models import Project, ProjectRelationship
49 
50 log = logging.getLogger(__name__)
51 
52 
53 def map_subproject_slug(view_func):
54     """
55     A decorator that maps a ``subproject_slug`` URL param into a Project.
56 
57     :raises: Http404 if the Project doesn't exist
58 
59     .. warning:: Does not take into account any kind of privacy settings.
60     """
61     @wraps(view_func)
62     def inner_view(request, subproject=None, subproject_slug=None, *args, **kwargs):  # noqa
63         if subproject is None and subproject_slug:
64             # Try to fetch by subproject alias first, otherwise we might end up
65             # redirected to an unrelated project.
66             try:
67                 # Depends on a project passed into kwargs
68                 rel = ProjectRelationship.objects.get(
69                     parent=kwargs['project'],
70                     alias=subproject_slug,
71                 )
72                 subproject = rel.child
73             except (ProjectRelationship.DoesNotExist, KeyError):
74                 subproject = get_object_or_404(Project, slug=subproject_slug)
75         return view_func(request, subproject=subproject, *args, **kwargs)
76 
77     return inner_view
78 
79 
80 def map_project_slug(view_func):
81     """
82     A decorator that maps a ``project_slug`` URL param into a Project.
83 
84     :raises: Http404 if the Project doesn't exist
85 
86     .. warning:: Does not take into account any kind of privacy settings.
87     """
88     @wraps(view_func)
89     def inner_view(request, project=None, project_slug=None, *args, **kwargs):  # noqa
90         if project is None:
91             if not project_slug:
92                 project_slug = request.slug
93             try:
94                 project = Project.objects.get(slug=project_slug)
95             except Project.DoesNotExist:
96                 raise Http404('Project does not exist.')
97         return view_func(request, project=project, *args, **kwargs)
98 
99     return inner_view
100 
101 
102 @map_project_slug
103 @map_subproject_slug
104 def redirect_project_slug(request, project, subproject):  # pylint: disable=unused-argument
105     """Handle / -> /en/latest/ directs on subdomains."""
106     return HttpResponseRedirect(resolve(subproject or project))
107 
108 
109 @map_project_slug
110 @map_subproject_slug
111 def redirect_page_with_filename(request, project, subproject, filename):  # pylint: disable=unused-argument  # noqa
112     """Redirect /page/file.html to /en/latest/file.html."""
113     return HttpResponseRedirect(
114         resolve(subproject or project, filename=filename))
115 
116 
117 def _serve_401(request, project):
118     res = render(request, '401.html')
119     res.status_code = 401
120     log.debug('Unauthorized access to {0} documentation'.format(project.slug))
121     return res
122 
123 
124 def _serve_file(request, filename, basepath):
125     # Serve the file from the proper location
126     if settings.DEBUG or getattr(settings, 'PYTHON_MEDIA', False):
127         # Serve from Python
128         return serve(request, filename, basepath)
129 
130     # Serve from Nginx
131     content_type, encoding = mimetypes.guess_type(
132         os.path.join(basepath, filename))
133     content_type = content_type or 'application/octet-stream'
134     response = HttpResponse(content_type=content_type)
135     if encoding:
136         response['Content-Encoding'] = encoding
137     try:
138         iri_path = os.path.join(
139             basepath[len(settings.SITE_ROOT):],
140             filename,
141         )
142         # NGINX does not support non-ASCII characters in the header, so we
143         # convert the IRI path to URI so it's compatible with what NGINX expects
144         # as the header value.
145         # https://github.com/benoitc/gunicorn/issues/1448
146         # https://docs.djangoproject.com/en/1.11/ref/unicode/#uri-and-iri-handling
147         x_accel_redirect = iri_to_uri(iri_path)
148         response['X-Accel-Redirect'] = x_accel_redirect
149     except UnicodeEncodeError:
150         raise Http404
151 
152     return response
153 
154 
155 @map_project_slug
156 @map_subproject_slug
157 def serve_docs(
158         request, project, subproject, lang_slug=None, version_slug=None,
159         filename=''):
160     """Exists to map existing proj, lang, version, filename views to the file format."""
161     if not version_slug:
162         version_slug = project.get_default_version()
163     try:
164         version = project.versions.public(request.user).get(slug=version_slug)
165     except Version.DoesNotExist:
166         # Properly raise a 404 if the version doesn't exist (or is inactive) and
167         # a 401 if it does
168         if project.versions.filter(slug=version_slug, active=True).exists():
169             return _serve_401(request, project)
170         raise Http404('Version does not exist.')
171     filename = resolve_path(
172         subproject or project,  # Resolve the subproject if it exists
173         version_slug=version_slug,
174         language=lang_slug,
175         filename=filename,
176         subdomain=True,  # subdomain will make it a "full" path without a URL prefix
177     )
178     if (version.privacy_level == constants.PRIVATE and
179             not AdminPermission.is_member(user=request.user, obj=project)):
180         return _serve_401(request, project)
181     return _serve_symlink_docs(
182         request,
183         filename=filename,
184         project=project,
185         privacy_level=version.privacy_level,
186     )
187 
188 
189 @map_project_slug
190 def _serve_symlink_docs(request, project, privacy_level, filename=''):
191     """Serve a file by symlink, or a 404 if not found."""
192     # Handle indexes
193     if filename == '' or filename[-1] == '/':
194         filename += 'index.html'
195 
196     # This breaks path joining, by ignoring the root when given an "absolute" path
197     if filename[0] == '/':
198         filename = filename[1:]
199 
200     log.info('Serving %s for %s', filename, project)
201 
202     files_tried = []
203 
204     serve_docs = getattr(settings, 'SERVE_DOCS', [constants.PRIVATE])
205 
206     if (settings.DEBUG or constants.PUBLIC in serve_docs) and privacy_level != constants.PRIVATE:  # yapf: disable  # noqa
207         public_symlink = PublicSymlink(project)
208         basepath = public_symlink.project_root
209         if os.path.exists(os.path.join(basepath, filename)):
210             return _serve_file(request, filename, basepath)
211 
212         files_tried.append(os.path.join(basepath, filename))
213 
214     if (settings.DEBUG or constants.PRIVATE in serve_docs) and privacy_level == constants.PRIVATE:  # yapf: disable  # noqa
215         # Handle private
216         private_symlink = PrivateSymlink(project)
217         basepath = private_symlink.project_root
218 
219         if os.path.exists(os.path.join(basepath, filename)):
220             return _serve_file(request, filename, basepath)
221 
222         files_tried.append(os.path.join(basepath, filename))
223 
224     raise Http404(
225         'File not found. Tried these files: %s' % ','.join(files_tried))
226
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/readthedocs/core/urls/subdomain.py b/readthedocs/core/urls/subdomain.py
--- a/readthedocs/core/urls/subdomain.py
+++ b/readthedocs/core/urls/subdomain.py
@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+
 """URL configurations for subdomains."""
 from __future__ import absolute_import
 
@@ -10,7 +12,7 @@
 
 from readthedocs.core.views.serve import (
     redirect_page_with_filename,
-    redirect_project_slug, serve_docs
+    redirect_project_slug, serve_docs, robots_txt,
 )
 from readthedocs.core.views import (
     server_error_500,
@@ -22,6 +24,8 @@
 handler404 = server_error_404
 
 subdomain_urls = [
+    url(r'robots.txt$', robots_txt, name='robots_txt'),
+
     url(r'^(?:|projects/(?P<subproject_slug>{project_slug})/)'
         r'page/(?P<filename>.*)$'.format(**pattern_opts),
         redirect_page_with_filename,
diff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py
--- a/readthedocs/core/views/serve.py
+++ b/readthedocs/core/views/serve.py
@@ -223,3 +223,49 @@
 
     raise Http404(
         'File not found. Tried these files: %s' % ','.join(files_tried))
+
+
+@map_project_slug
+def robots_txt(request, project):
+    """
+    Serve custom user's defined ``/robots.txt``.
+
+    If the user added a ``robots.txt`` in the "default version" of the project,
+    we serve it directly.
+    """
+    # Use the ``robots.txt`` file from the default version configured
+    version_slug = project.get_default_version()
+    version = project.versions.get(slug=version_slug)
+
+    no_serve_robots_txt = any([
+        # If project is private or,
+        project.privacy_level == constants.PRIVATE,
+        # default version is private or,
+        version.privacy_level == constants.PRIVATE,
+        # default version is not active or,
+        not version.active,
+        # default version is not built
+        not version.built,
+    ])
+    if no_serve_robots_txt:
+        # ... we do return a 404
+        raise Http404()
+
+    filename = resolve_path(
+        project,
+        version_slug=version_slug,
+        filename='robots.txt',
+        subdomain=True,  # subdomain will make it a "full" path without a URL prefix
+    )
+
+    # This breaks path joining, by ignoring the root when given an "absolute" path
+    if filename[0] == '/':
+        filename = filename[1:]
+
+    basepath = PublicSymlink(project).project_root
+    fullpath = os.path.join(basepath, filename)
+
+    if os.path.exists(fullpath):
+        return HttpResponse(open(fullpath).read(), content_type='text/plain')
+
+    return HttpResponse('User-agent: *\nAllow: /\n', content_type='text/plain')

{"golden_diff": "diff --git a/readthedocs/core/urls/subdomain.py b/readthedocs/core/urls/subdomain.py\n--- a/readthedocs/core/urls/subdomain.py\n+++ b/readthedocs/core/urls/subdomain.py\n@@ -1,3 +1,5 @@\n+# -*- coding: utf-8 -*-\n+\n \"\"\"URL configurations for subdomains.\"\"\"\n from __future__ import absolute_import\n \n@@ -10,7 +12,7 @@\n \n from readthedocs.core.views.serve import (\n     redirect_page_with_filename,\n-    redirect_project_slug, serve_docs\n+    redirect_project_slug, serve_docs, robots_txt,\n )\n from readthedocs.core.views import (\n     server_error_500,\n@@ -22,6 +24,8 @@\n handler404 = server_error_404\n \n subdomain_urls = [\n+    url(r'robots.txt$', robots_txt, name='robots_txt'),\n+\n     url(r'^(?:|projects/(?P<subproject_slug>{project_slug})/)'\n         r'page/(?P<filename>.*)$'.format(**pattern_opts),\n         redirect_page_with_filename,\ndiff --git a/readthedocs/core/views/serve.py b/readthedocs/core/views/serve.py\n--- a/readthedocs/core/views/serve.py\n+++ b/readthedocs/core/views/serve.py\n@@ -223,3 +223,49 @@\n \n     raise Http404(\n         'File not found. Tried these files: %s' % ','.join(files_tried))\n+\n+\n+@map_project_slug\n+def robots_txt(request, project):\n+    \"\"\"\n+    Serve custom user's defined ``/robots.txt``.\n+\n+    If the user added a ``robots.txt`` in the \"default version\" of the project,\n+    we serve it directly.\n+    \"\"\"\n+    # Use the ``robots.txt`` file from the default version configured\n+    version_slug = project.get_default_version()\n+    version = project.versions.get(slug=version_slug)\n+\n+    no_serve_robots_txt = any([\n+        # If project is private or,\n+        project.privacy_level == constants.PRIVATE,\n+        # default version is private or,\n+        version.privacy_level == constants.PRIVATE,\n+        # default version is not active or,\n+        not version.active,\n+        # default version is not built\n+        not version.built,\n+    ])\n+    if no_serve_robots_txt:\n+        # ... we do return a 404\n+        raise Http404()\n+\n+    filename = resolve_path(\n+        project,\n+        version_slug=version_slug,\n+        filename='robots.txt',\n+        subdomain=True,  # subdomain will make it a \"full\" path without a URL prefix\n+    )\n+\n+    # This breaks path joining, by ignoring the root when given an \"absolute\" path\n+    if filename[0] == '/':\n+        filename = filename[1:]\n+\n+    basepath = PublicSymlink(project).project_root\n+    fullpath = os.path.join(basepath, filename)\n+\n+    if os.path.exists(fullpath):\n+        return HttpResponse(open(fullpath).read(), content_type='text/plain')\n+\n+    return HttpResponse('User-agent: *\\nAllow: /\\n', content_type='text/plain')\n", "issue": "Custom robots.txt support?\nWe've talked about blowing away the protected designation, so not sure if it makes sense to put special case on the protected privacy level, but maybe a separate option for docs that shouldn't be crawled?\n", "before_files": [{"content": "\"\"\"URL configurations for subdomains.\"\"\"\nfrom __future__ import absolute_import\n\nfrom functools import reduce\nfrom operator import add\n\nfrom django.conf.urls import url\nfrom django.conf import settings\nfrom django.conf.urls.static import static\n\nfrom readthedocs.core.views.serve import (\n    redirect_page_with_filename,\n    redirect_project_slug, serve_docs\n)\nfrom readthedocs.core.views import (\n    server_error_500,\n    server_error_404,\n)\nfrom readthedocs.constants import pattern_opts\n\nhandler500 = server_error_500\nhandler404 = server_error_404\n\nsubdomain_urls = [\n    url(r'^(?:|projects/(?P<subproject_slug>{project_slug})/)'\n        r'page/(?P<filename>.*)$'.format(**pattern_opts),\n        redirect_page_with_filename,\n        name='docs_detail'),\n\n    url((r'^(?:|projects/(?P<subproject_slug>{project_slug})/)$').format(**pattern_opts),\n        redirect_project_slug,\n        name='redirect_project_slug'),\n\n    url((r'^(?:|projects/(?P<subproject_slug>{project_slug})/)'\n         r'(?P<lang_slug>{lang_slug})/'\n         r'(?P<version_slug>{version_slug})/'\n         r'(?P<filename>{filename_slug})$'.format(**pattern_opts)),\n        serve_docs,\n        name='docs_detail'),\n]\n\ngroups = [subdomain_urls]\n\n# Needed to serve media locally\nif getattr(settings, 'DEBUG', False):\n    groups.insert(0, static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT))\n\nurlpatterns = reduce(add, groups)\n", "path": "readthedocs/core/urls/subdomain.py"}, {"content": "# -*- coding: utf-8 -*-\n\"\"\"\nDoc serving from Python.\n\nIn production there are two modes,\n* Serving from public symlinks in nginx (readthedocs.org & readthedocs.com)\n* Serving from private symlinks in Python (readthedocs.com only)\n\nIn development, we have two modes:\n* Serving from public symlinks in Python\n* Serving from private symlinks in Python\n\nThis means we should only serve from public symlinks in dev,\nand generally default to serving from private symlinks in Python only.\n\nPrivacy\n-------\n\nThese views will take into account the version privacy level.\n\nSettings\n--------\n\nPYTHON_MEDIA (False) - Set this to True to serve docs & media from Python\nSERVE_DOCS (['private']) - The list of ['private', 'public'] docs to serve.\n\"\"\"\n\nfrom __future__ import (\n    absolute_import, division, print_function, unicode_literals)\n\nimport logging\nimport mimetypes\nimport os\nfrom functools import wraps\n\nfrom django.conf import settings\nfrom django.http import HttpResponse, HttpResponseRedirect, Http404\nfrom django.shortcuts import get_object_or_404\nfrom django.shortcuts import render\nfrom django.utils.encoding import iri_to_uri\nfrom django.views.static import serve\n\nfrom readthedocs.builds.models import Version\nfrom readthedocs.core.permissions import AdminPermission\nfrom readthedocs.core.resolver import resolve, resolve_path\nfrom readthedocs.core.symlink import PrivateSymlink, PublicSymlink\nfrom readthedocs.projects import constants\nfrom readthedocs.projects.models import Project, ProjectRelationship\n\nlog = logging.getLogger(__name__)\n\n\ndef map_subproject_slug(view_func):\n    \"\"\"\n    A decorator that maps a ``subproject_slug`` URL param into a Project.\n\n    :raises: Http404 if the Project doesn't exist\n\n    .. warning:: Does not take into account any kind of privacy settings.\n    \"\"\"\n    @wraps(view_func)\n    def inner_view(request, subproject=None, subproject_slug=None, *args, **kwargs):  # noqa\n        if subproject is None and subproject_slug:\n            # Try to fetch by subproject alias first, otherwise we might end up\n            # redirected to an unrelated project.\n            try:\n                # Depends on a project passed into kwargs\n                rel = ProjectRelationship.objects.get(\n                    parent=kwargs['project'],\n                    alias=subproject_slug,\n                )\n                subproject = rel.child\n            except (ProjectRelationship.DoesNotExist, KeyError):\n                subproject = get_object_or_404(Project, slug=subproject_slug)\n        return view_func(request, subproject=subproject, *args, **kwargs)\n\n    return inner_view\n\n\ndef map_project_slug(view_func):\n    \"\"\"\n    A decorator that maps a ``project_slug`` URL param into a Project.\n\n    :raises: Http404 if the Project doesn't exist\n\n    .. warning:: Does not take into account any kind of privacy settings.\n    \"\"\"\n    @wraps(view_func)\n    def inner_view(request, project=None, project_slug=None, *args, **kwargs):  # noqa\n        if project is None:\n            if not project_slug:\n                project_slug = request.slug\n            try:\n                project = Project.objects.get(slug=project_slug)\n            except Project.DoesNotExist:\n                raise Http404('Project does not exist.')\n        return view_func(request, project=project, *args, **kwargs)\n\n    return inner_view\n\n\n@map_project_slug\n@map_subproject_slug\ndef redirect_project_slug(request, project, subproject):  # pylint: disable=unused-argument\n    \"\"\"Handle / -> /en/latest/ directs on subdomains.\"\"\"\n    return HttpResponseRedirect(resolve(subproject or project))\n\n\n@map_project_slug\n@map_subproject_slug\ndef redirect_page_with_filename(request, project, subproject, filename):  # pylint: disable=unused-argument  # noqa\n    \"\"\"Redirect /page/file.html to /en/latest/file.html.\"\"\"\n    return HttpResponseRedirect(\n        resolve(subproject or project, filename=filename))\n\n\ndef _serve_401(request, project):\n    res = render(request, '401.html')\n    res.status_code = 401\n    log.debug('Unauthorized access to {0} documentation'.format(project.slug))\n    return res\n\n\ndef _serve_file(request, filename, basepath):\n    # Serve the file from the proper location\n    if settings.DEBUG or getattr(settings, 'PYTHON_MEDIA', False):\n        # Serve from Python\n        return serve(request, filename, basepath)\n\n    # Serve from Nginx\n    content_type, encoding = mimetypes.guess_type(\n        os.path.join(basepath, filename))\n    content_type = content_type or 'application/octet-stream'\n    response = HttpResponse(content_type=content_type)\n    if encoding:\n        response['Content-Encoding'] = encoding\n    try:\n        iri_path = os.path.join(\n            basepath[len(settings.SITE_ROOT):],\n            filename,\n        )\n        # NGINX does not support non-ASCII characters in the header, so we\n        # convert the IRI path to URI so it's compatible with what NGINX expects\n        # as the header value.\n        # https://github.com/benoitc/gunicorn/issues/1448\n        # https://docs.djangoproject.com/en/1.11/ref/unicode/#uri-and-iri-handling\n        x_accel_redirect = iri_to_uri(iri_path)\n        response['X-Accel-Redirect'] = x_accel_redirect\n    except UnicodeEncodeError:\n        raise Http404\n\n    return response\n\n\n@map_project_slug\n@map_subproject_slug\ndef serve_docs(\n        request, project, subproject, lang_slug=None, version_slug=None,\n        filename=''):\n    \"\"\"Exists to map existing proj, lang, version, filename views to the file format.\"\"\"\n    if not version_slug:\n        version_slug = project.get_default_version()\n    try:\n        version = project.versions.public(request.user).get(slug=version_slug)\n    except Version.DoesNotExist:\n        # Properly raise a 404 if the version doesn't exist (or is inactive) and\n        # a 401 if it does\n        if project.versions.filter(slug=version_slug, active=True).exists():\n            return _serve_401(request, project)\n        raise Http404('Version does not exist.')\n    filename = resolve_path(\n        subproject or project,  # Resolve the subproject if it exists\n        version_slug=version_slug,\n        language=lang_slug,\n        filename=filename,\n        subdomain=True,  # subdomain will make it a \"full\" path without a URL prefix\n    )\n    if (version.privacy_level == constants.PRIVATE and\n            not AdminPermission.is_member(user=request.user, obj=project)):\n        return _serve_401(request, project)\n    return _serve_symlink_docs(\n        request,\n        filename=filename,\n        project=project,\n        privacy_level=version.privacy_level,\n    )\n\n\n@map_project_slug\ndef _serve_symlink_docs(request, project, privacy_level, filename=''):\n    \"\"\"Serve a file by symlink, or a 404 if not found.\"\"\"\n    # Handle indexes\n    if filename == '' or filename[-1] == '/':\n        filename += 'index.html'\n\n    # This breaks path joining, by ignoring the root when given an \"absolute\" path\n    if filename[0] == '/':\n        filename = filename[1:]\n\n    log.info('Serving %s for %s', filename, project)\n\n    files_tried = []\n\n    serve_docs = getattr(settings, 'SERVE_DOCS', [constants.PRIVATE])\n\n    if (settings.DEBUG or constants.PUBLIC in serve_docs) and privacy_level != constants.PRIVATE:  # yapf: disable  # noqa\n        public_symlink = PublicSymlink(project)\n        basepath = public_symlink.project_root\n        if os.path.exists(os.path.join(basepath, filename)):\n            return _serve_file(request, filename, basepath)\n\n        files_tried.append(os.path.join(basepath, filename))\n\n    if (settings.DEBUG or constants.PRIVATE in serve_docs) and privacy_level == constants.PRIVATE:  # yapf: disable  # noqa\n        # Handle private\n        private_symlink = PrivateSymlink(project)\n        basepath = private_symlink.project_root\n\n        if os.path.exists(os.path.join(basepath, filename)):\n            return _serve_file(request, filename, basepath)\n\n        files_tried.append(os.path.join(basepath, filename))\n\n    raise Http404(\n        'File not found. Tried these files: %s' % ','.join(files_tried))\n", "path": "readthedocs/core/views/serve.py"}], "after_files": [{"content": "# -*- coding: utf-8 -*-\n\n\"\"\"URL configurations for subdomains.\"\"\"\nfrom __future__ import absolute_import\n\nfrom functools import reduce\nfrom operator import add\n\nfrom django.conf.urls import url\nfrom django.conf import settings\nfrom django.conf.urls.static import static\n\nfrom readthedocs.core.views.serve import (\n    redirect_page_with_filename,\n    redirect_project_slug, serve_docs, robots_txt,\n)\nfrom readthedocs.core.views import (\n    server_error_500,\n    server_error_404,\n)\nfrom readthedocs.constants import pattern_opts\n\nhandler500 = server_error_500\nhandler404 = server_error_404\n\nsubdomain_urls = [\n    url(r'robots.txt$', robots_txt, name='robots_txt'),\n\n    url(r'^(?:|projects/(?P<subproject_slug>{project_slug})/)'\n        r'page/(?P<filename>.*)$'.format(**pattern_opts),\n        redirect_page_with_filename,\n        name='docs_detail'),\n\n    url((r'^(?:|projects/(?P<subproject_slug>{project_slug})/)$').format(**pattern_opts),\n        redirect_project_slug,\n        name='redirect_project_slug'),\n\n    url((r'^(?:|projects/(?P<subproject_slug>{project_slug})/)'\n         r'(?P<lang_slug>{lang_slug})/'\n         r'(?P<version_slug>{version_slug})/'\n         r'(?P<filename>{filename_slug})$'.format(**pattern_opts)),\n        serve_docs,\n        name='docs_detail'),\n]\n\ngroups = [subdomain_urls]\n\n# Needed to serve media locally\nif getattr(settings, 'DEBUG', False):\n    groups.insert(0, static(settings.MEDIA_URL, document_root=settings.MEDIA_ROOT))\n\nurlpatterns = reduce(add, groups)\n", "path": "readthedocs/core/urls/subdomain.py"}, {"content": "# -*- coding: utf-8 -*-\n\"\"\"\nDoc serving from Python.\n\nIn production there are two modes,\n* Serving from public symlinks in nginx (readthedocs.org & readthedocs.com)\n* Serving from private symlinks in Python (readthedocs.com only)\n\nIn development, we have two modes:\n* Serving from public symlinks in Python\n* Serving from private symlinks in Python\n\nThis means we should only serve from public symlinks in dev,\nand generally default to serving from private symlinks in Python only.\n\nPrivacy\n-------\n\nThese views will take into account the version privacy level.\n\nSettings\n--------\n\nPYTHON_MEDIA (False) - Set this to True to serve docs & media from Python\nSERVE_DOCS (['private']) - The list of ['private', 'public'] docs to serve.\n\"\"\"\n\nfrom __future__ import (\n    absolute_import, division, print_function, unicode_literals)\n\nimport logging\nimport mimetypes\nimport os\nfrom functools import wraps\n\nfrom django.conf import settings\nfrom django.http import HttpResponse, HttpResponseRedirect, Http404\nfrom django.shortcuts import get_object_or_404\nfrom django.shortcuts import render\nfrom django.utils.encoding import iri_to_uri\nfrom django.views.static import serve\n\nfrom readthedocs.builds.models import Version\nfrom readthedocs.core.permissions import AdminPermission\nfrom readthedocs.core.resolver import resolve, resolve_path\nfrom readthedocs.core.symlink import PrivateSymlink, PublicSymlink\nfrom readthedocs.projects import constants\nfrom readthedocs.projects.models import Project, ProjectRelationship\n\nlog = logging.getLogger(__name__)\n\n\ndef map_subproject_slug(view_func):\n    \"\"\"\n    A decorator that maps a ``subproject_slug`` URL param into a Project.\n\n    :raises: Http404 if the Project doesn't exist\n\n    .. warning:: Does not take into account any kind of privacy settings.\n    \"\"\"\n    @wraps(view_func)\n    def inner_view(request, subproject=None, subproject_slug=None, *args, **kwargs):  # noqa\n        if subproject is None and subproject_slug:\n            # Try to fetch by subproject alias first, otherwise we might end up\n            # redirected to an unrelated project.\n            try:\n                # Depends on a project passed into kwargs\n                rel = ProjectRelationship.objects.get(\n                    parent=kwargs['project'],\n                    alias=subproject_slug,\n                )\n                subproject = rel.child\n            except (ProjectRelationship.DoesNotExist, KeyError):\n                subproject = get_object_or_404(Project, slug=subproject_slug)\n        return view_func(request, subproject=subproject, *args, **kwargs)\n\n    return inner_view\n\n\ndef map_project_slug(view_func):\n    \"\"\"\n    A decorator that maps a ``project_slug`` URL param into a Project.\n\n    :raises: Http404 if the Project doesn't exist\n\n    .. warning:: Does not take into account any kind of privacy settings.\n    \"\"\"\n    @wraps(view_func)\n    def inner_view(request, project=None, project_slug=None, *args, **kwargs):  # noqa\n        if project is None:\n            if not project_slug:\n                project_slug = request.slug\n            try:\n                project = Project.objects.get(slug=project_slug)\n            except Project.DoesNotExist:\n                raise Http404('Project does not exist.')\n        return view_func(request, project=project, *args, **kwargs)\n\n    return inner_view\n\n\n@map_project_slug\n@map_subproject_slug\ndef redirect_project_slug(request, project, subproject):  # pylint: disable=unused-argument\n    \"\"\"Handle / -> /en/latest/ directs on subdomains.\"\"\"\n    return HttpResponseRedirect(resolve(subproject or project))\n\n\n@map_project_slug\n@map_subproject_slug\ndef redirect_page_with_filename(request, project, subproject, filename):  # pylint: disable=unused-argument  # noqa\n    \"\"\"Redirect /page/file.html to /en/latest/file.html.\"\"\"\n    return HttpResponseRedirect(\n        resolve(subproject or project, filename=filename))\n\n\ndef _serve_401(request, project):\n    res = render(request, '401.html')\n    res.status_code = 401\n    log.debug('Unauthorized access to {0} documentation'.format(project.slug))\n    return res\n\n\ndef _serve_file(request, filename, basepath):\n    # Serve the file from the proper location\n    if settings.DEBUG or getattr(settings, 'PYTHON_MEDIA', False):\n        # Serve from Python\n        return serve(request, filename, basepath)\n\n    # Serve from Nginx\n    content_type, encoding = mimetypes.guess_type(\n        os.path.join(basepath, filename))\n    content_type = content_type or 'application/octet-stream'\n    response = HttpResponse(content_type=content_type)\n    if encoding:\n        response['Content-Encoding'] = encoding\n    try:\n        iri_path = os.path.join(\n            basepath[len(settings.SITE_ROOT):],\n            filename,\n        )\n        # NGINX does not support non-ASCII characters in the header, so we\n        # convert the IRI path to URI so it's compatible with what NGINX expects\n        # as the header value.\n        # https://github.com/benoitc/gunicorn/issues/1448\n        # https://docs.djangoproject.com/en/1.11/ref/unicode/#uri-and-iri-handling\n        x_accel_redirect = iri_to_uri(iri_path)\n        response['X-Accel-Redirect'] = x_accel_redirect\n    except UnicodeEncodeError:\n        raise Http404\n\n    return response\n\n\n@map_project_slug\n@map_subproject_slug\ndef serve_docs(\n        request, project, subproject, lang_slug=None, version_slug=None,\n        filename=''):\n    \"\"\"Exists to map existing proj, lang, version, filename views to the file format.\"\"\"\n    if not version_slug:\n        version_slug = project.get_default_version()\n    try:\n        version = project.versions.public(request.user).get(slug=version_slug)\n    except Version.DoesNotExist:\n        # Properly raise a 404 if the version doesn't exist (or is inactive) and\n        # a 401 if it does\n        if project.versions.filter(slug=version_slug, active=True).exists():\n            return _serve_401(request, project)\n        raise Http404('Version does not exist.')\n    filename = resolve_path(\n        subproject or project,  # Resolve the subproject if it exists\n        version_slug=version_slug,\n        language=lang_slug,\n        filename=filename,\n        subdomain=True,  # subdomain will make it a \"full\" path without a URL prefix\n    )\n    if (version.privacy_level == constants.PRIVATE and\n            not AdminPermission.is_member(user=request.user, obj=project)):\n        return _serve_401(request, project)\n    return _serve_symlink_docs(\n        request,\n        filename=filename,\n        project=project,\n        privacy_level=version.privacy_level,\n    )\n\n\n@map_project_slug\ndef _serve_symlink_docs(request, project, privacy_level, filename=''):\n    \"\"\"Serve a file by symlink, or a 404 if not found.\"\"\"\n    # Handle indexes\n    if filename == '' or filename[-1] == '/':\n        filename += 'index.html'\n\n    # This breaks path joining, by ignoring the root when given an \"absolute\" path\n    if filename[0] == '/':\n        filename = filename[1:]\n\n    log.info('Serving %s for %s', filename, project)\n\n    files_tried = []\n\n    serve_docs = getattr(settings, 'SERVE_DOCS', [constants.PRIVATE])\n\n    if (settings.DEBUG or constants.PUBLIC in serve_docs) and privacy_level != constants.PRIVATE:  # yapf: disable  # noqa\n        public_symlink = PublicSymlink(project)\n        basepath = public_symlink.project_root\n        if os.path.exists(os.path.join(basepath, filename)):\n            return _serve_file(request, filename, basepath)\n\n        files_tried.append(os.path.join(basepath, filename))\n\n    if (settings.DEBUG or constants.PRIVATE in serve_docs) and privacy_level == constants.PRIVATE:  # yapf: disable  # noqa\n        # Handle private\n        private_symlink = PrivateSymlink(project)\n        basepath = private_symlink.project_root\n\n        if os.path.exists(os.path.join(basepath, filename)):\n            return _serve_file(request, filename, basepath)\n\n        files_tried.append(os.path.join(basepath, filename))\n\n    raise Http404(\n        'File not found. Tried these files: %s' % ','.join(files_tried))\n\n\n@map_project_slug\ndef robots_txt(request, project):\n    \"\"\"\n    Serve custom user's defined ``/robots.txt``.\n\n    If the user added a ``robots.txt`` in the \"default version\" of the project,\n    we serve it directly.\n    \"\"\"\n    # Use the ``robots.txt`` file from the default version configured\n    version_slug = project.get_default_version()\n    version = project.versions.get(slug=version_slug)\n\n    no_serve_robots_txt = any([\n        # If project is private or,\n        project.privacy_level == constants.PRIVATE,\n        # default version is private or,\n        version.privacy_level == constants.PRIVATE,\n        # default version is not active or,\n        not version.active,\n        # default version is not built\n        not version.built,\n    ])\n    if no_serve_robots_txt:\n        # ... we do return a 404\n        raise Http404()\n\n    filename = resolve_path(\n        project,\n        version_slug=version_slug,\n        filename='robots.txt',\n        subdomain=True,  # subdomain will make it a \"full\" path without a URL prefix\n    )\n\n    # This breaks path joining, by ignoring the root when given an \"absolute\" path\n    if filename[0] == '/':\n        filename = filename[1:]\n\n    basepath = PublicSymlink(project).project_root\n    fullpath = os.path.join(basepath, filename)\n\n    if os.path.exists(fullpath):\n        return HttpResponse(open(fullpath).read(), content_type='text/plain')\n\n    return HttpResponse('User-agent: *\\nAllow: /\\n', content_type='text/plain')\n", "path": "readthedocs/core/views/serve.py"}]}