We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
bug 1314290: Update to Django 1.8.16
Django 1.8.15 -> 1.8.16 - Security release

https://www.djangoproject.com/weblog/2016/nov/01/security-releases/

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `kuma/api/v1/search/__init__.py`
Content:
```
1 from django import http
2 from django.conf import settings
3 from django.utils.cache import patch_cache_control
4 from elasticsearch import exceptions
5 from elasticsearch_dsl import Q, query, Search
6 from redo import retrying
7 
8 from kuma.api.v1.decorators import allow_CORS_GET
9 
10 from .forms import SearchForm
11 
12 # This is the number of seconds to be put into the Cache-Control max-age header
13 # if the search is successful.
14 # We can increase the number as we feel more and more comfortable with how
15 # the `/api/v1/search` works.
16 SEARCH_CACHE_CONTROL_MAX_AGE = 60 * 60 * 12
17 
18 
19 class JsonResponse(http.JsonResponse):
20     """The only reason this exists is so that other Django views can call
21     views that return instances of this and then get to the data before it
22     gets JSON serialized.
23     This is something that rest_framework's JsonResponse supports.
24     Ultimately, the only view that cares is the (old) Kuma search view page
25     that calls the view function here in this file. Now it can do something like:
26 
27         response = kuma.api.v1.search.search(request)
28         found = response.data
29 
30     """
31 
32     def __init__(self, data, *args, **kwargs):
33         self.data = data
34         super().__init__(data, *args, **kwargs)
35 
36 
37 def legacy(request, locale=None):
38     raise NotImplementedError("work harder")
39 
40 
41 @allow_CORS_GET
42 def search(request, locale=None):
43     initial = {"size": 10, "page": 1, "archive": SearchForm.ARCHIVE_CHOICES[0]}
44     if locale:
45         initial["locale"] = locale
46     form = SearchForm(request.GET, initial=initial)
47     if not form.is_valid():
48         return JsonResponse({"errors": form.errors.get_json_data()}, status=400)
49 
50     locales = form.cleaned_data["locale"] or [settings.LANGUAGE_CODE]
51     assert isinstance(locales, list)
52 
53     params = {
54         "locales": [x.lower() for x in locales],
55         "archive": form.cleaned_data["archive"],
56         "query": form.cleaned_data["q"],
57         "size": form.cleaned_data["size"],
58         "page": form.cleaned_data["page"],
59         "sort": form.cleaned_data["sort"],
60         # The `slug` is always stored, as a Keyword index, in lowercase.
61         "slug_prefixes": [x.lower() for x in form.cleaned_data["slug_prefix"]],
62     }
63 
64     # By default, assume that we will try to make suggestions.
65     make_suggestions = True
66     if len(params["query"]) > 100 or max(len(x) for x in params["query"].split()) > 30:
67         # For example, if it's a really long query, or a specific word is just too
68         # long, you can get those tricky
69         # TransportError(500, 'search_phase_execution_exception', 'Term too complex:
70         # errors which are hard to prevent against.
71         make_suggestions = False
72 
73     results = _find(
74         params,
75         make_suggestions=make_suggestions,
76     )
77     response = JsonResponse(results)
78 
79     # The reason for caching is that most of the time, the searches people make
80     # are short and often stand a high chance of being reused by other users
81     # in the CDN.
82     # The worst that can happen is that we fill up the CDN with cached responses
83     # that end up being stored there and never reused by another user.
84     # We could consider only bothering with this based on looking at the parameters.
85     # For example, if someone made a search with "complex parameters" we could skip
86     # cache-control because it'll just be a waste to store it (CDN and client).
87     # The reason for not using a "shared" cache-control, i.e. `s-max-age` is
88     # because the cache-control seconds we intend to set are appropriate for both
89     # the client and the CDN. If the value set is 3600 seconds, that means that
90     # clients might potentially re-use their own browser cache if they trigger
91     # a repeated search. And it's an appropriate number for the CDN too.
92     # For more info about how our search patterns behave,
93     # see https://github.com/mdn/kuma/issues/7799
94     patch_cache_control(response, public=True, max_age=SEARCH_CACHE_CONTROL_MAX_AGE)
95     return response
96 
97 
98 def _find(params, total_only=False, make_suggestions=False, min_suggestion_score=0.8):
99     search_query = Search(
100         index=settings.SEARCH_INDEX_NAME,
101     )
102     if make_suggestions:
103         # XXX research if it it's better to use phrase suggesters and if
104         # that works
105         # https://www.elastic.co/guide/en/elasticsearch/reference/current/search-suggesters.html#phrase-suggester
106         search_query = search_query.suggest(
107             "title_suggestions", params["query"], term={"field": "title"}
108         )
109         search_query = search_query.suggest(
110             "body_suggestions", params["query"], term={"field": "body"}
111         )
112 
113     sub_queries = []
114     sub_queries.append(Q("match", title={"query": params["query"], "boost": 2.0}))
115     sub_queries.append(Q("match", body={"query": params["query"], "boost": 1.0}))
116     if " " in params["query"]:
117         sub_queries.append(
118             Q("match_phrase", title={"query": params["query"], "boost": 10.0})
119         )
120         sub_queries.append(
121             Q("match_phrase", body={"query": params["query"], "boost": 5.0})
122         )
123 
124     sub_query = query.Bool(should=sub_queries)
125 
126     if params["locales"]:
127         search_query = search_query.filter("terms", locale=params["locales"])
128     if params["archive"] == "exclude":
129         search_query = search_query.filter("term", archived=False)
130     elif params["archive"] == "only":
131         search_query = search_query.filter("term", archived=True)
132 
133     if params["slug_prefixes"]:
134         sub_queries = [Q("prefix", slug=x) for x in params["slug_prefixes"]]
135         search_query = search_query.query(query.Bool(should=sub_queries))
136 
137     search_query = search_query.highlight_options(
138         pre_tags=["<mark>"],
139         post_tags=["</mark>"],
140         number_of_fragments=3,
141         fragment_size=120,
142         encoder="html",
143     )
144     search_query = search_query.highlight("title", "body")
145 
146     if params["sort"] == "relevance":
147         search_query = search_query.sort("_score", "-popularity")
148         search_query = search_query.query(sub_query)
149     elif params["sort"] == "popularity":
150         search_query = search_query.sort("-popularity", "_score")
151         search_query = search_query.query(sub_query)
152     else:
153         popularity_factor = 10.0
154         boost_mode = "sum"
155         score_mode = "max"
156         search_query = search_query.query(
157             "function_score",
158             query=sub_query,
159             functions=[
160                 query.SF(
161                     "field_value_factor",
162                     field="popularity",
163                     factor=popularity_factor,
164                     missing=0.0,
165                 )
166             ],
167             boost_mode=boost_mode,
168             score_mode=score_mode,
169         )
170 
171     search_query = search_query.source(excludes=["body"])
172 
173     search_query = search_query[
174         params["size"] * (params["page"] - 1) : params["size"] * params["page"]
175     ]
176 
177     retry_options = {
178         "retry_exceptions": (
179             # This is the standard operational exception.
180             exceptions.ConnectionError,
181             # This can happen if the search happened right as the index had
182             # just been deleted due to a fresh re-indexing happening in Yari.
183             exceptions.NotFoundError,
184             # This can happen when the index simply isn't ready yet.
185             exceptions.TransportError,
186         ),
187         # The default in redo is 60 seconds. Let's tone that down.
188         "sleeptime": settings.ES_RETRY_SLEEPTIME,
189         "attempts": settings.ES_RETRY_ATTEMPTS,
190         "jitter": settings.ES_RETRY_JITTER,
191     }
192     with retrying(search_query.execute, **retry_options) as retrying_function:
193         response = retrying_function()
194 
195     if total_only:
196         return response.hits.total
197 
198     metadata = {
199         "took_ms": response.took,
200         "total": {
201             # The `response.hits.total` is a `elasticsearch_dsl.utils.AttrDict`
202             # instance. Pluck only the exact data needed.
203             "value": response.hits.total.value,
204             "relation": response.hits.total.relation,
205         },
206         "size": params["size"],
207         "page": params["page"],
208     }
209     documents = []
210     for hit in response:
211         try:
212             body_highlight = list(hit.meta.highlight.body)
213         except AttributeError:
214             body_highlight = []
215         try:
216             title_highlight = list(hit.meta.highlight.title)
217         except AttributeError:
218             title_highlight = []
219 
220         d = {
221             "mdn_url": hit.meta.id,
222             "score": hit.meta.score,
223             "title": hit.title,
224             "locale": hit.locale,
225             "slug": hit.slug,
226             "popularity": hit.popularity,
227             "archived": hit.archived,
228             "summary": hit.summary,
229             "highlight": {
230                 "body": body_highlight,
231                 "title": title_highlight,
232             },
233         }
234         documents.append(d)
235 
236     try:
237         suggest = getattr(response, "suggest")
238     except AttributeError:
239         suggest = None
240 
241     suggestions = []
242     if suggest:
243         suggestion_strings = _unpack_suggestions(
244             params["query"],
245             response.suggest,
246             ("body_suggestions", "title_suggestions"),
247         )
248 
249         for score, string in suggestion_strings:
250             if score > min_suggestion_score or 1:
251                 # Sure, this is different way to spell, but what will it yield
252                 # if you actually search it?
253                 total = _find(dict(params, query=string), total_only=True)
254                 if total["value"] > 0:
255                     suggestions.append(
256                         {
257                             "text": string,
258                             "total": {
259                                 # This 'total' is an `AttrDict` instance.
260                                 "value": total.value,
261                                 "relation": total.relation,
262                             },
263                         }
264                     )
265                     # Since they're sorted by score, it's usually never useful
266                     # to suggestion more than exactly 1 good suggestion.
267                     break
268 
269     return {
270         "documents": documents,
271         "metadata": metadata,
272         "suggestions": suggestions,
273     }
274 
275 
276 def _unpack_suggestions(query, suggest, keys):
277     alternatives = []
278     for key in keys:
279         for suggestion in getattr(suggest, key, []):
280             for option in suggestion.options:
281                 alternatives.append(
282                     (
283                         option.score,
284                         query[0 : suggestion.offset]
285                         + option.text
286                         + query[suggestion.offset + suggestion.length :],
287                     )
288                 )
289     alternatives.sort(reverse=True)  # highest score first
290     return alternatives
291
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/kuma/api/v1/search/__init__.py b/kuma/api/v1/search/__init__.py
--- a/kuma/api/v1/search/__init__.py
+++ b/kuma/api/v1/search/__init__.py
@@ -110,15 +110,29 @@
             "body_suggestions", params["query"], term={"field": "body"}
         )
 
+    # The business logic here that we search for things different ways,
+    # and each different way as a different boost which dictates its importance.
+    # The importance order is as follows:
+    #
+    #  1. Title match-phrase
+    #  2. Title match
+    #  3. Body match-phrase
+    #  4. Body match
+    #
+    # The order is determined by the `boost` number in the code below.
+    # Remember that sort order is a combination of "match" and popularity, but
+    # ideally the popularity should complement. Try to get a pretty good
+    # sort by pure relevance first, and let popularity just make it better.
+    #
     sub_queries = []
-    sub_queries.append(Q("match", title={"query": params["query"], "boost": 2.0}))
+    sub_queries.append(Q("match", title={"query": params["query"], "boost": 5.0}))
     sub_queries.append(Q("match", body={"query": params["query"], "boost": 1.0}))
     if " " in params["query"]:
         sub_queries.append(
             Q("match_phrase", title={"query": params["query"], "boost": 10.0})
         )
         sub_queries.append(
-            Q("match_phrase", body={"query": params["query"], "boost": 5.0})
+            Q("match_phrase", body={"query": params["query"], "boost": 2.0})
         )
 
     sub_query = query.Bool(should=sub_queries)

{"golden_diff": "diff --git a/kuma/api/v1/search/__init__.py b/kuma/api/v1/search/__init__.py\n--- a/kuma/api/v1/search/__init__.py\n+++ b/kuma/api/v1/search/__init__.py\n@@ -110,15 +110,29 @@\n             \"body_suggestions\", params[\"query\"], term={\"field\": \"body\"}\n         )\n \n+    # The business logic here that we search for things different ways,\n+    # and each different way as a different boost which dictates its importance.\n+    # The importance order is as follows:\n+    #\n+    #  1. Title match-phrase\n+    #  2. Title match\n+    #  3. Body match-phrase\n+    #  4. Body match\n+    #\n+    # The order is determined by the `boost` number in the code below.\n+    # Remember that sort order is a combination of \"match\" and popularity, but\n+    # ideally the popularity should complement. Try to get a pretty good\n+    # sort by pure relevance first, and let popularity just make it better.\n+    #\n     sub_queries = []\n-    sub_queries.append(Q(\"match\", title={\"query\": params[\"query\"], \"boost\": 2.0}))\n+    sub_queries.append(Q(\"match\", title={\"query\": params[\"query\"], \"boost\": 5.0}))\n     sub_queries.append(Q(\"match\", body={\"query\": params[\"query\"], \"boost\": 1.0}))\n     if \" \" in params[\"query\"]:\n         sub_queries.append(\n             Q(\"match_phrase\", title={\"query\": params[\"query\"], \"boost\": 10.0})\n         )\n         sub_queries.append(\n-            Q(\"match_phrase\", body={\"query\": params[\"query\"], \"boost\": 5.0})\n+            Q(\"match_phrase\", body={\"query\": params[\"query\"], \"boost\": 2.0})\n         )\n \n     sub_query = query.Bool(should=sub_queries)\n", "issue": "bug 1314290: Update to Django 1.8.16\nDjango 1.8.15 -> 1.8.16 - Security release\r\n\r\nhttps://www.djangoproject.com/weblog/2016/nov/01/security-releases/\n", "before_files": [{"content": "from django import http\nfrom django.conf import settings\nfrom django.utils.cache import patch_cache_control\nfrom elasticsearch import exceptions\nfrom elasticsearch_dsl import Q, query, Search\nfrom redo import retrying\n\nfrom kuma.api.v1.decorators import allow_CORS_GET\n\nfrom .forms import SearchForm\n\n# This is the number of seconds to be put into the Cache-Control max-age header\n# if the search is successful.\n# We can increase the number as we feel more and more comfortable with how\n# the `/api/v1/search` works.\nSEARCH_CACHE_CONTROL_MAX_AGE = 60 * 60 * 12\n\n\nclass JsonResponse(http.JsonResponse):\n    \"\"\"The only reason this exists is so that other Django views can call\n    views that return instances of this and then get to the data before it\n    gets JSON serialized.\n    This is something that rest_framework's JsonResponse supports.\n    Ultimately, the only view that cares is the (old) Kuma search view page\n    that calls the view function here in this file. Now it can do something like:\n\n        response = kuma.api.v1.search.search(request)\n        found = response.data\n\n    \"\"\"\n\n    def __init__(self, data, *args, **kwargs):\n        self.data = data\n        super().__init__(data, *args, **kwargs)\n\n\ndef legacy(request, locale=None):\n    raise NotImplementedError(\"work harder\")\n\n\n@allow_CORS_GET\ndef search(request, locale=None):\n    initial = {\"size\": 10, \"page\": 1, \"archive\": SearchForm.ARCHIVE_CHOICES[0]}\n    if locale:\n        initial[\"locale\"] = locale\n    form = SearchForm(request.GET, initial=initial)\n    if not form.is_valid():\n        return JsonResponse({\"errors\": form.errors.get_json_data()}, status=400)\n\n    locales = form.cleaned_data[\"locale\"] or [settings.LANGUAGE_CODE]\n    assert isinstance(locales, list)\n\n    params = {\n        \"locales\": [x.lower() for x in locales],\n        \"archive\": form.cleaned_data[\"archive\"],\n        \"query\": form.cleaned_data[\"q\"],\n        \"size\": form.cleaned_data[\"size\"],\n        \"page\": form.cleaned_data[\"page\"],\n        \"sort\": form.cleaned_data[\"sort\"],\n        # The `slug` is always stored, as a Keyword index, in lowercase.\n        \"slug_prefixes\": [x.lower() for x in form.cleaned_data[\"slug_prefix\"]],\n    }\n\n    # By default, assume that we will try to make suggestions.\n    make_suggestions = True\n    if len(params[\"query\"]) > 100 or max(len(x) for x in params[\"query\"].split()) > 30:\n        # For example, if it's a really long query, or a specific word is just too\n        # long, you can get those tricky\n        # TransportError(500, 'search_phase_execution_exception', 'Term too complex:\n        # errors which are hard to prevent against.\n        make_suggestions = False\n\n    results = _find(\n        params,\n        make_suggestions=make_suggestions,\n    )\n    response = JsonResponse(results)\n\n    # The reason for caching is that most of the time, the searches people make\n    # are short and often stand a high chance of being reused by other users\n    # in the CDN.\n    # The worst that can happen is that we fill up the CDN with cached responses\n    # that end up being stored there and never reused by another user.\n    # We could consider only bothering with this based on looking at the parameters.\n    # For example, if someone made a search with \"complex parameters\" we could skip\n    # cache-control because it'll just be a waste to store it (CDN and client).\n    # The reason for not using a \"shared\" cache-control, i.e. `s-max-age` is\n    # because the cache-control seconds we intend to set are appropriate for both\n    # the client and the CDN. If the value set is 3600 seconds, that means that\n    # clients might potentially re-use their own browser cache if they trigger\n    # a repeated search. And it's an appropriate number for the CDN too.\n    # For more info about how our search patterns behave,\n    # see https://github.com/mdn/kuma/issues/7799\n    patch_cache_control(response, public=True, max_age=SEARCH_CACHE_CONTROL_MAX_AGE)\n    return response\n\n\ndef _find(params, total_only=False, make_suggestions=False, min_suggestion_score=0.8):\n    search_query = Search(\n        index=settings.SEARCH_INDEX_NAME,\n    )\n    if make_suggestions:\n        # XXX research if it it's better to use phrase suggesters and if\n        # that works\n        # https://www.elastic.co/guide/en/elasticsearch/reference/current/search-suggesters.html#phrase-suggester\n        search_query = search_query.suggest(\n            \"title_suggestions\", params[\"query\"], term={\"field\": \"title\"}\n        )\n        search_query = search_query.suggest(\n            \"body_suggestions\", params[\"query\"], term={\"field\": \"body\"}\n        )\n\n    sub_queries = []\n    sub_queries.append(Q(\"match\", title={\"query\": params[\"query\"], \"boost\": 2.0}))\n    sub_queries.append(Q(\"match\", body={\"query\": params[\"query\"], \"boost\": 1.0}))\n    if \" \" in params[\"query\"]:\n        sub_queries.append(\n            Q(\"match_phrase\", title={\"query\": params[\"query\"], \"boost\": 10.0})\n        )\n        sub_queries.append(\n            Q(\"match_phrase\", body={\"query\": params[\"query\"], \"boost\": 5.0})\n        )\n\n    sub_query = query.Bool(should=sub_queries)\n\n    if params[\"locales\"]:\n        search_query = search_query.filter(\"terms\", locale=params[\"locales\"])\n    if params[\"archive\"] == \"exclude\":\n        search_query = search_query.filter(\"term\", archived=False)\n    elif params[\"archive\"] == \"only\":\n        search_query = search_query.filter(\"term\", archived=True)\n\n    if params[\"slug_prefixes\"]:\n        sub_queries = [Q(\"prefix\", slug=x) for x in params[\"slug_prefixes\"]]\n        search_query = search_query.query(query.Bool(should=sub_queries))\n\n    search_query = search_query.highlight_options(\n        pre_tags=[\"<mark>\"],\n        post_tags=[\"</mark>\"],\n        number_of_fragments=3,\n        fragment_size=120,\n        encoder=\"html\",\n    )\n    search_query = search_query.highlight(\"title\", \"body\")\n\n    if params[\"sort\"] == \"relevance\":\n        search_query = search_query.sort(\"_score\", \"-popularity\")\n        search_query = search_query.query(sub_query)\n    elif params[\"sort\"] == \"popularity\":\n        search_query = search_query.sort(\"-popularity\", \"_score\")\n        search_query = search_query.query(sub_query)\n    else:\n        popularity_factor = 10.0\n        boost_mode = \"sum\"\n        score_mode = \"max\"\n        search_query = search_query.query(\n            \"function_score\",\n            query=sub_query,\n            functions=[\n                query.SF(\n                    \"field_value_factor\",\n                    field=\"popularity\",\n                    factor=popularity_factor,\n                    missing=0.0,\n                )\n            ],\n            boost_mode=boost_mode,\n            score_mode=score_mode,\n        )\n\n    search_query = search_query.source(excludes=[\"body\"])\n\n    search_query = search_query[\n        params[\"size\"] * (params[\"page\"] - 1) : params[\"size\"] * params[\"page\"]\n    ]\n\n    retry_options = {\n        \"retry_exceptions\": (\n            # This is the standard operational exception.\n            exceptions.ConnectionError,\n            # This can happen if the search happened right as the index had\n            # just been deleted due to a fresh re-indexing happening in Yari.\n            exceptions.NotFoundError,\n            # This can happen when the index simply isn't ready yet.\n            exceptions.TransportError,\n        ),\n        # The default in redo is 60 seconds. Let's tone that down.\n        \"sleeptime\": settings.ES_RETRY_SLEEPTIME,\n        \"attempts\": settings.ES_RETRY_ATTEMPTS,\n        \"jitter\": settings.ES_RETRY_JITTER,\n    }\n    with retrying(search_query.execute, **retry_options) as retrying_function:\n        response = retrying_function()\n\n    if total_only:\n        return response.hits.total\n\n    metadata = {\n        \"took_ms\": response.took,\n        \"total\": {\n            # The `response.hits.total` is a `elasticsearch_dsl.utils.AttrDict`\n            # instance. Pluck only the exact data needed.\n            \"value\": response.hits.total.value,\n            \"relation\": response.hits.total.relation,\n        },\n        \"size\": params[\"size\"],\n        \"page\": params[\"page\"],\n    }\n    documents = []\n    for hit in response:\n        try:\n            body_highlight = list(hit.meta.highlight.body)\n        except AttributeError:\n            body_highlight = []\n        try:\n            title_highlight = list(hit.meta.highlight.title)\n        except AttributeError:\n            title_highlight = []\n\n        d = {\n            \"mdn_url\": hit.meta.id,\n            \"score\": hit.meta.score,\n            \"title\": hit.title,\n            \"locale\": hit.locale,\n            \"slug\": hit.slug,\n            \"popularity\": hit.popularity,\n            \"archived\": hit.archived,\n            \"summary\": hit.summary,\n            \"highlight\": {\n                \"body\": body_highlight,\n                \"title\": title_highlight,\n            },\n        }\n        documents.append(d)\n\n    try:\n        suggest = getattr(response, \"suggest\")\n    except AttributeError:\n        suggest = None\n\n    suggestions = []\n    if suggest:\n        suggestion_strings = _unpack_suggestions(\n            params[\"query\"],\n            response.suggest,\n            (\"body_suggestions\", \"title_suggestions\"),\n        )\n\n        for score, string in suggestion_strings:\n            if score > min_suggestion_score or 1:\n                # Sure, this is different way to spell, but what will it yield\n                # if you actually search it?\n                total = _find(dict(params, query=string), total_only=True)\n                if total[\"value\"] > 0:\n                    suggestions.append(\n                        {\n                            \"text\": string,\n                            \"total\": {\n                                # This 'total' is an `AttrDict` instance.\n                                \"value\": total.value,\n                                \"relation\": total.relation,\n                            },\n                        }\n                    )\n                    # Since they're sorted by score, it's usually never useful\n                    # to suggestion more than exactly 1 good suggestion.\n                    break\n\n    return {\n        \"documents\": documents,\n        \"metadata\": metadata,\n        \"suggestions\": suggestions,\n    }\n\n\ndef _unpack_suggestions(query, suggest, keys):\n    alternatives = []\n    for key in keys:\n        for suggestion in getattr(suggest, key, []):\n            for option in suggestion.options:\n                alternatives.append(\n                    (\n                        option.score,\n                        query[0 : suggestion.offset]\n                        + option.text\n                        + query[suggestion.offset + suggestion.length :],\n                    )\n                )\n    alternatives.sort(reverse=True)  # highest score first\n    return alternatives\n", "path": "kuma/api/v1/search/__init__.py"}], "after_files": [{"content": "from django import http\nfrom django.conf import settings\nfrom django.utils.cache import patch_cache_control\nfrom elasticsearch import exceptions\nfrom elasticsearch_dsl import Q, query, Search\nfrom redo import retrying\n\nfrom kuma.api.v1.decorators import allow_CORS_GET\n\nfrom .forms import SearchForm\n\n# This is the number of seconds to be put into the Cache-Control max-age header\n# if the search is successful.\n# We can increase the number as we feel more and more comfortable with how\n# the `/api/v1/search` works.\nSEARCH_CACHE_CONTROL_MAX_AGE = 60 * 60 * 12\n\n\nclass JsonResponse(http.JsonResponse):\n    \"\"\"The only reason this exists is so that other Django views can call\n    views that return instances of this and then get to the data before it\n    gets JSON serialized.\n    This is something that rest_framework's JsonResponse supports.\n    Ultimately, the only view that cares is the (old) Kuma search view page\n    that calls the view function here in this file. Now it can do something like:\n\n        response = kuma.api.v1.search.search(request)\n        found = response.data\n\n    \"\"\"\n\n    def __init__(self, data, *args, **kwargs):\n        self.data = data\n        super().__init__(data, *args, **kwargs)\n\n\ndef legacy(request, locale=None):\n    raise NotImplementedError(\"work harder\")\n\n\n@allow_CORS_GET\ndef search(request, locale=None):\n    initial = {\"size\": 10, \"page\": 1, \"archive\": SearchForm.ARCHIVE_CHOICES[0]}\n    if locale:\n        initial[\"locale\"] = locale\n    form = SearchForm(request.GET, initial=initial)\n    if not form.is_valid():\n        return JsonResponse({\"errors\": form.errors.get_json_data()}, status=400)\n\n    locales = form.cleaned_data[\"locale\"] or [settings.LANGUAGE_CODE]\n    assert isinstance(locales, list)\n\n    params = {\n        \"locales\": [x.lower() for x in locales],\n        \"archive\": form.cleaned_data[\"archive\"],\n        \"query\": form.cleaned_data[\"q\"],\n        \"size\": form.cleaned_data[\"size\"],\n        \"page\": form.cleaned_data[\"page\"],\n        \"sort\": form.cleaned_data[\"sort\"],\n        # The `slug` is always stored, as a Keyword index, in lowercase.\n        \"slug_prefixes\": [x.lower() for x in form.cleaned_data[\"slug_prefix\"]],\n    }\n\n    # By default, assume that we will try to make suggestions.\n    make_suggestions = True\n    if len(params[\"query\"]) > 100 or max(len(x) for x in params[\"query\"].split()) > 30:\n        # For example, if it's a really long query, or a specific word is just too\n        # long, you can get those tricky\n        # TransportError(500, 'search_phase_execution_exception', 'Term too complex:\n        # errors which are hard to prevent against.\n        make_suggestions = False\n\n    results = _find(\n        params,\n        make_suggestions=make_suggestions,\n    )\n    response = JsonResponse(results)\n\n    # The reason for caching is that most of the time, the searches people make\n    # are short and often stand a high chance of being reused by other users\n    # in the CDN.\n    # The worst that can happen is that we fill up the CDN with cached responses\n    # that end up being stored there and never reused by another user.\n    # We could consider only bothering with this based on looking at the parameters.\n    # For example, if someone made a search with \"complex parameters\" we could skip\n    # cache-control because it'll just be a waste to store it (CDN and client).\n    # The reason for not using a \"shared\" cache-control, i.e. `s-max-age` is\n    # because the cache-control seconds we intend to set are appropriate for both\n    # the client and the CDN. If the value set is 3600 seconds, that means that\n    # clients might potentially re-use their own browser cache if they trigger\n    # a repeated search. And it's an appropriate number for the CDN too.\n    # For more info about how our search patterns behave,\n    # see https://github.com/mdn/kuma/issues/7799\n    patch_cache_control(response, public=True, max_age=SEARCH_CACHE_CONTROL_MAX_AGE)\n    return response\n\n\ndef _find(params, total_only=False, make_suggestions=False, min_suggestion_score=0.8):\n    search_query = Search(\n        index=settings.SEARCH_INDEX_NAME,\n    )\n    if make_suggestions:\n        # XXX research if it it's better to use phrase suggesters and if\n        # that works\n        # https://www.elastic.co/guide/en/elasticsearch/reference/current/search-suggesters.html#phrase-suggester\n        search_query = search_query.suggest(\n            \"title_suggestions\", params[\"query\"], term={\"field\": \"title\"}\n        )\n        search_query = search_query.suggest(\n            \"body_suggestions\", params[\"query\"], term={\"field\": \"body\"}\n        )\n\n    # The business logic here that we search for things different ways,\n    # and each different way as a different boost which dictates its importance.\n    # The importance order is as follows:\n    #\n    #  1. Title match-phrase\n    #  2. Title match\n    #  3. Body match-phrase\n    #  4. Body match\n    #\n    # The order is determined by the `boost` number in the code below.\n    # Remember that sort order is a combination of \"match\" and popularity, but\n    # ideally the popularity should complement. Try to get a pretty good\n    # sort by pure relevance first, and let popularity just make it better.\n    #\n    sub_queries = []\n    sub_queries.append(Q(\"match\", title={\"query\": params[\"query\"], \"boost\": 5.0}))\n    sub_queries.append(Q(\"match\", body={\"query\": params[\"query\"], \"boost\": 1.0}))\n    if \" \" in params[\"query\"]:\n        sub_queries.append(\n            Q(\"match_phrase\", title={\"query\": params[\"query\"], \"boost\": 10.0})\n        )\n        sub_queries.append(\n            Q(\"match_phrase\", body={\"query\": params[\"query\"], \"boost\": 2.0})\n        )\n\n    sub_query = query.Bool(should=sub_queries)\n\n    if params[\"locales\"]:\n        search_query = search_query.filter(\"terms\", locale=params[\"locales\"])\n    if params[\"archive\"] == \"exclude\":\n        search_query = search_query.filter(\"term\", archived=False)\n    elif params[\"archive\"] == \"only\":\n        search_query = search_query.filter(\"term\", archived=True)\n\n    if params[\"slug_prefixes\"]:\n        sub_queries = [Q(\"prefix\", slug=x) for x in params[\"slug_prefixes\"]]\n        search_query = search_query.query(query.Bool(should=sub_queries))\n\n    search_query = search_query.highlight_options(\n        pre_tags=[\"<mark>\"],\n        post_tags=[\"</mark>\"],\n        number_of_fragments=3,\n        fragment_size=120,\n        encoder=\"html\",\n    )\n    search_query = search_query.highlight(\"title\", \"body\")\n\n    if params[\"sort\"] == \"relevance\":\n        search_query = search_query.sort(\"_score\", \"-popularity\")\n        search_query = search_query.query(sub_query)\n    elif params[\"sort\"] == \"popularity\":\n        search_query = search_query.sort(\"-popularity\", \"_score\")\n        search_query = search_query.query(sub_query)\n    else:\n        popularity_factor = 10.0\n        boost_mode = \"sum\"\n        score_mode = \"max\"\n        search_query = search_query.query(\n            \"function_score\",\n            query=sub_query,\n            functions=[\n                query.SF(\n                    \"field_value_factor\",\n                    field=\"popularity\",\n                    factor=popularity_factor,\n                    missing=0.0,\n                )\n            ],\n            boost_mode=boost_mode,\n            score_mode=score_mode,\n        )\n\n    search_query = search_query.source(excludes=[\"body\"])\n\n    search_query = search_query[\n        params[\"size\"] * (params[\"page\"] - 1) : params[\"size\"] * params[\"page\"]\n    ]\n\n    retry_options = {\n        \"retry_exceptions\": (\n            # This is the standard operational exception.\n            exceptions.ConnectionError,\n            # This can happen if the search happened right as the index had\n            # just been deleted due to a fresh re-indexing happening in Yari.\n            exceptions.NotFoundError,\n            # This can happen when the index simply isn't ready yet.\n            exceptions.TransportError,\n        ),\n        # The default in redo is 60 seconds. Let's tone that down.\n        \"sleeptime\": settings.ES_RETRY_SLEEPTIME,\n        \"attempts\": settings.ES_RETRY_ATTEMPTS,\n        \"jitter\": settings.ES_RETRY_JITTER,\n    }\n    with retrying(search_query.execute, **retry_options) as retrying_function:\n        response = retrying_function()\n\n    if total_only:\n        return response.hits.total\n\n    metadata = {\n        \"took_ms\": response.took,\n        \"total\": {\n            # The `response.hits.total` is a `elasticsearch_dsl.utils.AttrDict`\n            # instance. Pluck only the exact data needed.\n            \"value\": response.hits.total.value,\n            \"relation\": response.hits.total.relation,\n        },\n        \"size\": params[\"size\"],\n        \"page\": params[\"page\"],\n    }\n    documents = []\n    for hit in response:\n        try:\n            body_highlight = list(hit.meta.highlight.body)\n        except AttributeError:\n            body_highlight = []\n        try:\n            title_highlight = list(hit.meta.highlight.title)\n        except AttributeError:\n            title_highlight = []\n\n        d = {\n            \"mdn_url\": hit.meta.id,\n            \"score\": hit.meta.score,\n            \"title\": hit.title,\n            \"locale\": hit.locale,\n            \"slug\": hit.slug,\n            \"popularity\": hit.popularity,\n            \"archived\": hit.archived,\n            \"summary\": hit.summary,\n            \"highlight\": {\n                \"body\": body_highlight,\n                \"title\": title_highlight,\n            },\n        }\n        documents.append(d)\n\n    try:\n        suggest = getattr(response, \"suggest\")\n    except AttributeError:\n        suggest = None\n\n    suggestions = []\n    if suggest:\n        suggestion_strings = _unpack_suggestions(\n            params[\"query\"],\n            response.suggest,\n            (\"body_suggestions\", \"title_suggestions\"),\n        )\n\n        for score, string in suggestion_strings:\n            if score > min_suggestion_score or 1:\n                # Sure, this is different way to spell, but what will it yield\n                # if you actually search it?\n                total = _find(dict(params, query=string), total_only=True)\n                if total[\"value\"] > 0:\n                    suggestions.append(\n                        {\n                            \"text\": string,\n                            \"total\": {\n                                # This 'total' is an `AttrDict` instance.\n                                \"value\": total.value,\n                                \"relation\": total.relation,\n                            },\n                        }\n                    )\n                    # Since they're sorted by score, it's usually never useful\n                    # to suggestion more than exactly 1 good suggestion.\n                    break\n\n    return {\n        \"documents\": documents,\n        \"metadata\": metadata,\n        \"suggestions\": suggestions,\n    }\n\n\ndef _unpack_suggestions(query, suggest, keys):\n    alternatives = []\n    for key in keys:\n        for suggestion in getattr(suggest, key, []):\n            for option in suggestion.options:\n                alternatives.append(\n                    (\n                        option.score,\n                        query[0 : suggestion.offset]\n                        + option.text\n                        + query[suggestion.offset + suggestion.length :],\n                    )\n                )\n    alternatives.sort(reverse=True)  # highest score first\n    return alternatives\n", "path": "kuma/api/v1/search/__init__.py"}]}