We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Search relevancy is still not ideal
#1020 fixed #1019 for the majority of packages, however a few still produce odd results:

For example:
https://warehouse.python.org/search/?q=flask (`Flask` package is 2nd, `Flask-Admin` is first)
https://warehouse.python.org/search/?q=django (`Django` package is 11th, `dotulu` is first)
https://warehouse.python.org/search/?q=git (First 3 packages do not have "git" anywhere in them)

This is hard to test in dev because the dev DB is a snapshot of TestPyPI, and those packages are missing.

@dstufft, would it be possible to get a more complete DB for local development?


--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `warehouse/packaging/search.py`
Content:
```
1 # Licensed under the Apache License, Version 2.0 (the "License");
2 # you may not use this file except in compliance with the License.
3 # You may obtain a copy of the License at
4 #
5 # http://www.apache.org/licenses/LICENSE-2.0
6 #
7 # Unless required by applicable law or agreed to in writing, software
8 # distributed under the License is distributed on an "AS IS" BASIS,
9 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10 # See the License for the specific language governing permissions and
11 # limitations under the License.
12 
13 from elasticsearch_dsl import DocType, String, analyzer, MetaField, Date
14 
15 from warehouse.search import doc_type
16 
17 
18 EmailAnalyzer = analyzer(
19     "email",
20     tokenizer="uax_url_email",
21     filter=["standard", "lowercase", "stop", "snowball"],
22 )
23 
24 
25 @doc_type
26 class Project(DocType):
27 
28     name = String()
29     normalized_name = String(index="not_analyzed")
30     version = String(index="not_analyzed", multi=True)
31     summary = String(analyzer="snowball")
32     description = String(analyzer="snowball")
33     author = String()
34     author_email = String(analyzer=EmailAnalyzer)
35     maintainer = String()
36     maintainer_email = String(analyzer=EmailAnalyzer)
37     license = String()
38     home_page = String(index="not_analyzed")
39     download_url = String(index="not_analyzed")
40     keywords = String(analyzer="snowball")
41     platform = String(index="not_analyzed")
42     created = Date()
43     classifiers = String(index="not_analyzed", multi=True)
44 
45     class Meta:
46         # disable the _all field to save some space
47         all = MetaField(enabled=False)
48 
49     @classmethod
50     def from_db(cls, release):
51         obj = cls(meta={"id": release.project.normalized_name})
52         obj["name"] = release.project.name
53         obj["normalized_name"] = release.project.normalized_name
54         obj["version"] = [r.version for r in release.project.releases]
55         obj["summary"] = release.summary
56         obj["description"] = release.description
57         obj["author"] = release.author
58         obj["author_email"] = release.author_email
59         obj["maintainer"] = release.maintainer
60         obj["maintainer_email"] = release.maintainer_email
61         obj["home_page"] = release.home_page
62         obj["download_url"] = release.download_url
63         obj["keywords"] = release.keywords
64         obj["platform"] = release.platform
65         obj["created"] = release.created
66         obj["classifiers"] = list(release.classifiers)
67 
68         return obj
69
```

Path: `warehouse/views.py`
Content:
```
1 # Licensed under the Apache License, Version 2.0 (the "License");
2 # you may not use this file except in compliance with the License.
3 # You may obtain a copy of the License at
4 #
5 # http://www.apache.org/licenses/LICENSE-2.0
6 #
7 # Unless required by applicable law or agreed to in writing, software
8 # distributed under the License is distributed on an "AS IS" BASIS,
9 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
10 # See the License for the specific language governing permissions and
11 # limitations under the License.
12 
13 import collections
14 
15 from pyramid.httpexceptions import (
16     HTTPException, HTTPSeeOther, HTTPMovedPermanently, HTTPNotFound,
17 )
18 from pyramid.view import (
19     notfound_view_config, forbidden_view_config, view_config,
20 )
21 from sqlalchemy import func
22 from sqlalchemy.orm import aliased, joinedload
23 
24 from warehouse.accounts import REDIRECT_FIELD_NAME
25 from warehouse.accounts.models import User
26 from warehouse.cache.origin import origin_cache
27 from warehouse.cache.http import cache_control
28 from warehouse.classifiers.models import Classifier
29 from warehouse.packaging.models import Project, Release, File
30 from warehouse.utils.row_counter import RowCount
31 from warehouse.utils.paginate import ElasticsearchPage, paginate_url_factory
32 
33 
34 @view_config(context=HTTPException)
35 @notfound_view_config(append_slash=HTTPMovedPermanently)
36 def httpexception_view(exc, request):
37     return exc
38 
39 
40 @forbidden_view_config()
41 def forbidden(exc, request):
42     # If the forbidden error is because the user isn't logged in, then we'll
43     # redirect them to the log in page.
44     if request.authenticated_userid is None:
45         url = request.route_url(
46             "accounts.login",
47             _query={REDIRECT_FIELD_NAME: request.path_qs},
48         )
49         return HTTPSeeOther(url)
50 
51     # If we've reached here, then the user is logged in and they are genuinely
52     # not allowed to access this page.
53     # TODO: Style the forbidden page.
54     return exc
55 
56 
57 @view_config(
58     route_name="robots.txt",
59     renderer="robots.txt",
60     decorator=[
61         cache_control(1 * 24 * 60 * 60),         # 1 day
62         origin_cache(
63             1 * 24 * 60 * 60,                    # 1 day
64             stale_while_revalidate=6 * 60 * 60,  # 6 hours
65             stale_if_error=1 * 24 * 60 * 60,     # 1 day
66         ),
67     ],
68 )
69 def robotstxt(request):
70     request.response.content_type = "text/plain"
71     return {}
72 
73 
74 @view_config(
75     route_name="index",
76     renderer="index.html",
77     decorator=[
78         origin_cache(
79             1 * 60 * 60,                      # 1 hour
80             stale_while_revalidate=10 * 60,   # 10 minutes
81             stale_if_error=1 * 24 * 60 * 60,  # 1 day
82             keys=["all-projects"],
83         ),
84     ]
85 )
86 def index(request):
87     project_names = [
88         r[0] for r in (
89             request.db.query(File.name)
90                    .group_by(File.name)
91                    .order_by(func.sum(File.downloads).desc())
92                    .limit(5)
93                    .all())
94     ]
95     release_a = aliased(
96         Release,
97         request.db.query(Release)
98                   .distinct(Release.name)
99                   .filter(Release.name.in_(project_names))
100                   .order_by(Release.name, Release._pypi_ordering.desc())
101                   .subquery(),
102     )
103     top_projects = (
104         request.db.query(release_a)
105                .options(joinedload(release_a.project),
106                         joinedload(release_a.uploader))
107                .order_by(func.array_idx(project_names, release_a.name))
108                .all()
109     )
110 
111     latest_releases = (
112         request.db.query(Release)
113                   .options(joinedload(Release.project),
114                            joinedload(Release.uploader))
115                   .order_by(Release.created.desc())
116                   .limit(5)
117                   .all()
118     )
119 
120     counts = dict(
121         request.db.query(RowCount.table_name, RowCount.count)
122                   .filter(
123                       RowCount.table_name.in_([
124                           Project.__tablename__,
125                           Release.__tablename__,
126                           File.__tablename__,
127                           User.__tablename__,
128                       ]))
129                   .all()
130     )
131 
132     return {
133         "latest_releases": latest_releases,
134         "top_projects": top_projects,
135         "num_projects": counts.get(Project.__tablename__, 0),
136         "num_releases": counts.get(Release.__tablename__, 0),
137         "num_files": counts.get(File.__tablename__, 0),
138         "num_users": counts.get(User.__tablename__, 0),
139     }
140 
141 
142 @view_config(
143     route_name="search",
144     renderer="search/results.html",
145     decorator=[
146         origin_cache(
147             1 * 60 * 60,                      # 1 hour
148             stale_while_revalidate=10 * 60,   # 10 minutes
149             stale_if_error=1 * 24 * 60 * 60,  # 1 day
150             keys=["all-projects"],
151         )
152     ],
153 )
154 def search(request):
155     if request.params.get("q"):
156         query = request.es.query(
157             "multi_match",
158             query=request.params["q"],
159             fields=[
160                 "name^2", "version", "author", "author_email", "maintainer",
161                 "maintainer_email", "home_page", "license", "summary",
162                 "description", "keywords", "platform", "download_url",
163             ],
164         ).suggest(
165             name="name_suggestion",
166             text=request.params["q"],
167             term={"field": "name"}
168         )
169     else:
170         query = request.es.query()
171 
172     if request.params.get("o"):
173         query = query.sort(request.params["o"])
174 
175     if request.params.getall("c"):
176         query = query.filter('terms', classifiers=request.params.getall("c"))
177 
178     page_num = int(request.params.get("page", 1))
179     page = ElasticsearchPage(
180         query,
181         page=page_num,
182         url_maker=paginate_url_factory(request),
183     )
184 
185     if page.page_count and page_num > page.page_count:
186         raise HTTPNotFound
187 
188     available_filters = collections.defaultdict(list)
189 
190     for cls in request.db.query(Classifier).order_by(Classifier.classifier):
191         first, *_ = cls.classifier.split(' :: ')
192         available_filters[first].append(cls.classifier)
193 
194     return {
195         "page": page,
196         "term": request.params.get("q", ''),
197         "order": request.params.get("o", ''),
198         "available_filters": sorted(available_filters.items()),
199         "applied_filters": request.params.getall("c"),
200     }
201 
202 
203 @view_config(
204     route_name="includes.current-user-indicator",
205     renderer="includes/current-user-indicator.html",
206     uses_session=True,
207 )
208 def current_user_indicator(request):
209     return {}
210 
211 
212 @view_config(route_name="health", renderer="string")
213 def health(request):
214     # This will ensure that we can access the database and run queries against
215     # it without doing anything that will take a lock or block other queries.
216     request.db.execute("SELECT 1")
217 
218     # Nothing will actually check this, but it's a little nicer to have
219     # something to return besides an empty body.
220     return "OK"
221
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/warehouse/packaging/search.py b/warehouse/packaging/search.py
--- a/warehouse/packaging/search.py
+++ b/warehouse/packaging/search.py
@@ -21,12 +21,18 @@
     filter=["standard", "lowercase", "stop", "snowball"],
 )
 
+NameAnalyzer = analyzer(
+    "normalized_name",
+    tokenizer="lowercase",
+    filter=["standard", "lowercase", "word_delimiter"],
+)
+
 
 @doc_type
 class Project(DocType):
 
     name = String()
-    normalized_name = String(index="not_analyzed")
+    normalized_name = String(analyzer=NameAnalyzer, index_options="docs")
     version = String(index="not_analyzed", multi=True)
     summary = String(analyzer="snowball")
     description = String(analyzer="snowball")
diff --git a/warehouse/views.py b/warehouse/views.py
--- a/warehouse/views.py
+++ b/warehouse/views.py
@@ -157,9 +157,10 @@
             "multi_match",
             query=request.params["q"],
             fields=[
-                "name^2", "version", "author", "author_email", "maintainer",
-                "maintainer_email", "home_page", "license", "summary",
-                "description", "keywords", "platform", "download_url",
+                "author", "author_email", "description^5", "download_url",
+                "home_page", "keywords^5", "license", "maintainer",
+                "maintainer_email", "normalized_name^10", "platform",
+                "summary^5",
             ],
         ).suggest(
             name="name_suggestion",

{"golden_diff": "diff --git a/warehouse/packaging/search.py b/warehouse/packaging/search.py\n--- a/warehouse/packaging/search.py\n+++ b/warehouse/packaging/search.py\n@@ -21,12 +21,18 @@\n     filter=[\"standard\", \"lowercase\", \"stop\", \"snowball\"],\n )\n \n+NameAnalyzer = analyzer(\n+    \"normalized_name\",\n+    tokenizer=\"lowercase\",\n+    filter=[\"standard\", \"lowercase\", \"word_delimiter\"],\n+)\n+\n \n @doc_type\n class Project(DocType):\n \n     name = String()\n-    normalized_name = String(index=\"not_analyzed\")\n+    normalized_name = String(analyzer=NameAnalyzer, index_options=\"docs\")\n     version = String(index=\"not_analyzed\", multi=True)\n     summary = String(analyzer=\"snowball\")\n     description = String(analyzer=\"snowball\")\ndiff --git a/warehouse/views.py b/warehouse/views.py\n--- a/warehouse/views.py\n+++ b/warehouse/views.py\n@@ -157,9 +157,10 @@\n             \"multi_match\",\n             query=request.params[\"q\"],\n             fields=[\n-                \"name^2\", \"version\", \"author\", \"author_email\", \"maintainer\",\n-                \"maintainer_email\", \"home_page\", \"license\", \"summary\",\n-                \"description\", \"keywords\", \"platform\", \"download_url\",\n+                \"author\", \"author_email\", \"description^5\", \"download_url\",\n+                \"home_page\", \"keywords^5\", \"license\", \"maintainer\",\n+                \"maintainer_email\", \"normalized_name^10\", \"platform\",\n+                \"summary^5\",\n             ],\n         ).suggest(\n             name=\"name_suggestion\",\n", "issue": "Search relevancy is still not ideal\n#1020 fixed #1019 for the majority of packages, however a few still produce odd results:\n\nFor example:\nhttps://warehouse.python.org/search/?q=flask (`Flask` package is 2nd, `Flask-Admin` is first)\nhttps://warehouse.python.org/search/?q=django (`Django` package is 11th, `dotulu` is first)\nhttps://warehouse.python.org/search/?q=git (First 3 packages do not have \"git\" anywhere in them)\n\nThis is hard to test in dev because the dev DB is a snapshot of TestPyPI, and those packages are missing.\n\n@dstufft, would it be possible to get a more complete DB for local development?\n\n", "before_files": [{"content": "# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom elasticsearch_dsl import DocType, String, analyzer, MetaField, Date\n\nfrom warehouse.search import doc_type\n\n\nEmailAnalyzer = analyzer(\n    \"email\",\n    tokenizer=\"uax_url_email\",\n    filter=[\"standard\", \"lowercase\", \"stop\", \"snowball\"],\n)\n\n\n@doc_type\nclass Project(DocType):\n\n    name = String()\n    normalized_name = String(index=\"not_analyzed\")\n    version = String(index=\"not_analyzed\", multi=True)\n    summary = String(analyzer=\"snowball\")\n    description = String(analyzer=\"snowball\")\n    author = String()\n    author_email = String(analyzer=EmailAnalyzer)\n    maintainer = String()\n    maintainer_email = String(analyzer=EmailAnalyzer)\n    license = String()\n    home_page = String(index=\"not_analyzed\")\n    download_url = String(index=\"not_analyzed\")\n    keywords = String(analyzer=\"snowball\")\n    platform = String(index=\"not_analyzed\")\n    created = Date()\n    classifiers = String(index=\"not_analyzed\", multi=True)\n\n    class Meta:\n        # disable the _all field to save some space\n        all = MetaField(enabled=False)\n\n    @classmethod\n    def from_db(cls, release):\n        obj = cls(meta={\"id\": release.project.normalized_name})\n        obj[\"name\"] = release.project.name\n        obj[\"normalized_name\"] = release.project.normalized_name\n        obj[\"version\"] = [r.version for r in release.project.releases]\n        obj[\"summary\"] = release.summary\n        obj[\"description\"] = release.description\n        obj[\"author\"] = release.author\n        obj[\"author_email\"] = release.author_email\n        obj[\"maintainer\"] = release.maintainer\n        obj[\"maintainer_email\"] = release.maintainer_email\n        obj[\"home_page\"] = release.home_page\n        obj[\"download_url\"] = release.download_url\n        obj[\"keywords\"] = release.keywords\n        obj[\"platform\"] = release.platform\n        obj[\"created\"] = release.created\n        obj[\"classifiers\"] = list(release.classifiers)\n\n        return obj\n", "path": "warehouse/packaging/search.py"}, {"content": "# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport collections\n\nfrom pyramid.httpexceptions import (\n    HTTPException, HTTPSeeOther, HTTPMovedPermanently, HTTPNotFound,\n)\nfrom pyramid.view import (\n    notfound_view_config, forbidden_view_config, view_config,\n)\nfrom sqlalchemy import func\nfrom sqlalchemy.orm import aliased, joinedload\n\nfrom warehouse.accounts import REDIRECT_FIELD_NAME\nfrom warehouse.accounts.models import User\nfrom warehouse.cache.origin import origin_cache\nfrom warehouse.cache.http import cache_control\nfrom warehouse.classifiers.models import Classifier\nfrom warehouse.packaging.models import Project, Release, File\nfrom warehouse.utils.row_counter import RowCount\nfrom warehouse.utils.paginate import ElasticsearchPage, paginate_url_factory\n\n\n@view_config(context=HTTPException)\n@notfound_view_config(append_slash=HTTPMovedPermanently)\ndef httpexception_view(exc, request):\n    return exc\n\n\n@forbidden_view_config()\ndef forbidden(exc, request):\n    # If the forbidden error is because the user isn't logged in, then we'll\n    # redirect them to the log in page.\n    if request.authenticated_userid is None:\n        url = request.route_url(\n            \"accounts.login\",\n            _query={REDIRECT_FIELD_NAME: request.path_qs},\n        )\n        return HTTPSeeOther(url)\n\n    # If we've reached here, then the user is logged in and they are genuinely\n    # not allowed to access this page.\n    # TODO: Style the forbidden page.\n    return exc\n\n\n@view_config(\n    route_name=\"robots.txt\",\n    renderer=\"robots.txt\",\n    decorator=[\n        cache_control(1 * 24 * 60 * 60),         # 1 day\n        origin_cache(\n            1 * 24 * 60 * 60,                    # 1 day\n            stale_while_revalidate=6 * 60 * 60,  # 6 hours\n            stale_if_error=1 * 24 * 60 * 60,     # 1 day\n        ),\n    ],\n)\ndef robotstxt(request):\n    request.response.content_type = \"text/plain\"\n    return {}\n\n\n@view_config(\n    route_name=\"index\",\n    renderer=\"index.html\",\n    decorator=[\n        origin_cache(\n            1 * 60 * 60,                      # 1 hour\n            stale_while_revalidate=10 * 60,   # 10 minutes\n            stale_if_error=1 * 24 * 60 * 60,  # 1 day\n            keys=[\"all-projects\"],\n        ),\n    ]\n)\ndef index(request):\n    project_names = [\n        r[0] for r in (\n            request.db.query(File.name)\n                   .group_by(File.name)\n                   .order_by(func.sum(File.downloads).desc())\n                   .limit(5)\n                   .all())\n    ]\n    release_a = aliased(\n        Release,\n        request.db.query(Release)\n                  .distinct(Release.name)\n                  .filter(Release.name.in_(project_names))\n                  .order_by(Release.name, Release._pypi_ordering.desc())\n                  .subquery(),\n    )\n    top_projects = (\n        request.db.query(release_a)\n               .options(joinedload(release_a.project),\n                        joinedload(release_a.uploader))\n               .order_by(func.array_idx(project_names, release_a.name))\n               .all()\n    )\n\n    latest_releases = (\n        request.db.query(Release)\n                  .options(joinedload(Release.project),\n                           joinedload(Release.uploader))\n                  .order_by(Release.created.desc())\n                  .limit(5)\n                  .all()\n    )\n\n    counts = dict(\n        request.db.query(RowCount.table_name, RowCount.count)\n                  .filter(\n                      RowCount.table_name.in_([\n                          Project.__tablename__,\n                          Release.__tablename__,\n                          File.__tablename__,\n                          User.__tablename__,\n                      ]))\n                  .all()\n    )\n\n    return {\n        \"latest_releases\": latest_releases,\n        \"top_projects\": top_projects,\n        \"num_projects\": counts.get(Project.__tablename__, 0),\n        \"num_releases\": counts.get(Release.__tablename__, 0),\n        \"num_files\": counts.get(File.__tablename__, 0),\n        \"num_users\": counts.get(User.__tablename__, 0),\n    }\n\n\n@view_config(\n    route_name=\"search\",\n    renderer=\"search/results.html\",\n    decorator=[\n        origin_cache(\n            1 * 60 * 60,                      # 1 hour\n            stale_while_revalidate=10 * 60,   # 10 minutes\n            stale_if_error=1 * 24 * 60 * 60,  # 1 day\n            keys=[\"all-projects\"],\n        )\n    ],\n)\ndef search(request):\n    if request.params.get(\"q\"):\n        query = request.es.query(\n            \"multi_match\",\n            query=request.params[\"q\"],\n            fields=[\n                \"name^2\", \"version\", \"author\", \"author_email\", \"maintainer\",\n                \"maintainer_email\", \"home_page\", \"license\", \"summary\",\n                \"description\", \"keywords\", \"platform\", \"download_url\",\n            ],\n        ).suggest(\n            name=\"name_suggestion\",\n            text=request.params[\"q\"],\n            term={\"field\": \"name\"}\n        )\n    else:\n        query = request.es.query()\n\n    if request.params.get(\"o\"):\n        query = query.sort(request.params[\"o\"])\n\n    if request.params.getall(\"c\"):\n        query = query.filter('terms', classifiers=request.params.getall(\"c\"))\n\n    page_num = int(request.params.get(\"page\", 1))\n    page = ElasticsearchPage(\n        query,\n        page=page_num,\n        url_maker=paginate_url_factory(request),\n    )\n\n    if page.page_count and page_num > page.page_count:\n        raise HTTPNotFound\n\n    available_filters = collections.defaultdict(list)\n\n    for cls in request.db.query(Classifier).order_by(Classifier.classifier):\n        first, *_ = cls.classifier.split(' :: ')\n        available_filters[first].append(cls.classifier)\n\n    return {\n        \"page\": page,\n        \"term\": request.params.get(\"q\", ''),\n        \"order\": request.params.get(\"o\", ''),\n        \"available_filters\": sorted(available_filters.items()),\n        \"applied_filters\": request.params.getall(\"c\"),\n    }\n\n\n@view_config(\n    route_name=\"includes.current-user-indicator\",\n    renderer=\"includes/current-user-indicator.html\",\n    uses_session=True,\n)\ndef current_user_indicator(request):\n    return {}\n\n\n@view_config(route_name=\"health\", renderer=\"string\")\ndef health(request):\n    # This will ensure that we can access the database and run queries against\n    # it without doing anything that will take a lock or block other queries.\n    request.db.execute(\"SELECT 1\")\n\n    # Nothing will actually check this, but it's a little nicer to have\n    # something to return besides an empty body.\n    return \"OK\"\n", "path": "warehouse/views.py"}], "after_files": [{"content": "# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nfrom elasticsearch_dsl import DocType, String, analyzer, MetaField, Date\n\nfrom warehouse.search import doc_type\n\n\nEmailAnalyzer = analyzer(\n    \"email\",\n    tokenizer=\"uax_url_email\",\n    filter=[\"standard\", \"lowercase\", \"stop\", \"snowball\"],\n)\n\nNameAnalyzer = analyzer(\n    \"normalized_name\",\n    tokenizer=\"lowercase\",\n    filter=[\"standard\", \"lowercase\", \"word_delimiter\"],\n)\n\n\n@doc_type\nclass Project(DocType):\n\n    name = String()\n    normalized_name = String(analyzer=NameAnalyzer, index_options=\"docs\")\n    version = String(index=\"not_analyzed\", multi=True)\n    summary = String(analyzer=\"snowball\")\n    description = String(analyzer=\"snowball\")\n    author = String()\n    author_email = String(analyzer=EmailAnalyzer)\n    maintainer = String()\n    maintainer_email = String(analyzer=EmailAnalyzer)\n    license = String()\n    home_page = String(index=\"not_analyzed\")\n    download_url = String(index=\"not_analyzed\")\n    keywords = String(analyzer=\"snowball\")\n    platform = String(index=\"not_analyzed\")\n    created = Date()\n    classifiers = String(index=\"not_analyzed\", multi=True)\n\n    class Meta:\n        # disable the _all field to save some space\n        all = MetaField(enabled=False)\n\n    @classmethod\n    def from_db(cls, release):\n        obj = cls(meta={\"id\": release.project.normalized_name})\n        obj[\"name\"] = release.project.name\n        obj[\"normalized_name\"] = release.project.normalized_name\n        obj[\"version\"] = [r.version for r in release.project.releases]\n        obj[\"summary\"] = release.summary\n        obj[\"description\"] = release.description\n        obj[\"author\"] = release.author\n        obj[\"author_email\"] = release.author_email\n        obj[\"maintainer\"] = release.maintainer\n        obj[\"maintainer_email\"] = release.maintainer_email\n        obj[\"home_page\"] = release.home_page\n        obj[\"download_url\"] = release.download_url\n        obj[\"keywords\"] = release.keywords\n        obj[\"platform\"] = release.platform\n        obj[\"created\"] = release.created\n        obj[\"classifiers\"] = list(release.classifiers)\n\n        return obj\n", "path": "warehouse/packaging/search.py"}, {"content": "# Licensed under the Apache License, Version 2.0 (the \"License\");\n# you may not use this file except in compliance with the License.\n# You may obtain a copy of the License at\n#\n# http://www.apache.org/licenses/LICENSE-2.0\n#\n# Unless required by applicable law or agreed to in writing, software\n# distributed under the License is distributed on an \"AS IS\" BASIS,\n# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n# See the License for the specific language governing permissions and\n# limitations under the License.\n\nimport collections\n\nfrom pyramid.httpexceptions import (\n    HTTPException, HTTPSeeOther, HTTPMovedPermanently, HTTPNotFound,\n)\nfrom pyramid.view import (\n    notfound_view_config, forbidden_view_config, view_config,\n)\nfrom sqlalchemy import func\nfrom sqlalchemy.orm import aliased, joinedload\n\nfrom warehouse.accounts import REDIRECT_FIELD_NAME\nfrom warehouse.accounts.models import User\nfrom warehouse.cache.origin import origin_cache\nfrom warehouse.cache.http import cache_control\nfrom warehouse.classifiers.models import Classifier\nfrom warehouse.packaging.models import Project, Release, File\nfrom warehouse.utils.row_counter import RowCount\nfrom warehouse.utils.paginate import ElasticsearchPage, paginate_url_factory\n\n\n@view_config(context=HTTPException)\n@notfound_view_config(append_slash=HTTPMovedPermanently)\ndef httpexception_view(exc, request):\n    return exc\n\n\n@forbidden_view_config()\ndef forbidden(exc, request):\n    # If the forbidden error is because the user isn't logged in, then we'll\n    # redirect them to the log in page.\n    if request.authenticated_userid is None:\n        url = request.route_url(\n            \"accounts.login\",\n            _query={REDIRECT_FIELD_NAME: request.path_qs},\n        )\n        return HTTPSeeOther(url)\n\n    # If we've reached here, then the user is logged in and they are genuinely\n    # not allowed to access this page.\n    # TODO: Style the forbidden page.\n    return exc\n\n\n@view_config(\n    route_name=\"robots.txt\",\n    renderer=\"robots.txt\",\n    decorator=[\n        cache_control(1 * 24 * 60 * 60),         # 1 day\n        origin_cache(\n            1 * 24 * 60 * 60,                    # 1 day\n            stale_while_revalidate=6 * 60 * 60,  # 6 hours\n            stale_if_error=1 * 24 * 60 * 60,     # 1 day\n        ),\n    ],\n)\ndef robotstxt(request):\n    request.response.content_type = \"text/plain\"\n    return {}\n\n\n@view_config(\n    route_name=\"index\",\n    renderer=\"index.html\",\n    decorator=[\n        origin_cache(\n            1 * 60 * 60,                      # 1 hour\n            stale_while_revalidate=10 * 60,   # 10 minutes\n            stale_if_error=1 * 24 * 60 * 60,  # 1 day\n            keys=[\"all-projects\"],\n        ),\n    ]\n)\ndef index(request):\n    project_names = [\n        r[0] for r in (\n            request.db.query(File.name)\n                   .group_by(File.name)\n                   .order_by(func.sum(File.downloads).desc())\n                   .limit(5)\n                   .all())\n    ]\n    release_a = aliased(\n        Release,\n        request.db.query(Release)\n                  .distinct(Release.name)\n                  .filter(Release.name.in_(project_names))\n                  .order_by(Release.name, Release._pypi_ordering.desc())\n                  .subquery(),\n    )\n    top_projects = (\n        request.db.query(release_a)\n               .options(joinedload(release_a.project),\n                        joinedload(release_a.uploader))\n               .order_by(func.array_idx(project_names, release_a.name))\n               .all()\n    )\n\n    latest_releases = (\n        request.db.query(Release)\n                  .options(joinedload(Release.project),\n                           joinedload(Release.uploader))\n                  .order_by(Release.created.desc())\n                  .limit(5)\n                  .all()\n    )\n\n    counts = dict(\n        request.db.query(RowCount.table_name, RowCount.count)\n                  .filter(\n                      RowCount.table_name.in_([\n                          Project.__tablename__,\n                          Release.__tablename__,\n                          File.__tablename__,\n                          User.__tablename__,\n                      ]))\n                  .all()\n    )\n\n    return {\n        \"latest_releases\": latest_releases,\n        \"top_projects\": top_projects,\n        \"num_projects\": counts.get(Project.__tablename__, 0),\n        \"num_releases\": counts.get(Release.__tablename__, 0),\n        \"num_files\": counts.get(File.__tablename__, 0),\n        \"num_users\": counts.get(User.__tablename__, 0),\n    }\n\n\n@view_config(\n    route_name=\"search\",\n    renderer=\"search/results.html\",\n    decorator=[\n        origin_cache(\n            1 * 60 * 60,                      # 1 hour\n            stale_while_revalidate=10 * 60,   # 10 minutes\n            stale_if_error=1 * 24 * 60 * 60,  # 1 day\n            keys=[\"all-projects\"],\n        )\n    ],\n)\ndef search(request):\n    if request.params.get(\"q\"):\n        query = request.es.query(\n            \"multi_match\",\n            query=request.params[\"q\"],\n            fields=[\n                \"author\", \"author_email\", \"description^5\", \"download_url\",\n                \"home_page\", \"keywords^5\", \"license\", \"maintainer\",\n                \"maintainer_email\", \"normalized_name^10\", \"platform\",\n                \"summary^5\",\n            ],\n        ).suggest(\n            name=\"name_suggestion\",\n            text=request.params[\"q\"],\n            term={\"field\": \"name\"}\n        )\n    else:\n        query = request.es.query()\n\n    if request.params.get(\"o\"):\n        query = query.sort(request.params[\"o\"])\n\n    if request.params.getall(\"c\"):\n        query = query.filter('terms', classifiers=request.params.getall(\"c\"))\n\n    page_num = int(request.params.get(\"page\", 1))\n    page = ElasticsearchPage(\n        query,\n        page=page_num,\n        url_maker=paginate_url_factory(request),\n    )\n\n    if page.page_count and page_num > page.page_count:\n        raise HTTPNotFound\n\n    available_filters = collections.defaultdict(list)\n\n    for cls in request.db.query(Classifier).order_by(Classifier.classifier):\n        first, *_ = cls.classifier.split(' :: ')\n        available_filters[first].append(cls.classifier)\n\n    return {\n        \"page\": page,\n        \"term\": request.params.get(\"q\", ''),\n        \"order\": request.params.get(\"o\", ''),\n        \"available_filters\": sorted(available_filters.items()),\n        \"applied_filters\": request.params.getall(\"c\"),\n    }\n\n\n@view_config(\n    route_name=\"includes.current-user-indicator\",\n    renderer=\"includes/current-user-indicator.html\",\n    uses_session=True,\n)\ndef current_user_indicator(request):\n    return {}\n\n\n@view_config(route_name=\"health\", renderer=\"string\")\ndef health(request):\n    # This will ensure that we can access the database and run queries against\n    # it without doing anything that will take a lock or block other queries.\n    request.db.execute(\"SELECT 1\")\n\n    # Nothing will actually check this, but it's a little nicer to have\n    # something to return besides an empty body.\n    return \"OK\"\n", "path": "warehouse/views.py"}]}