We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
Reindex dangerously excludes slugs
The reindexing exludes these slugs: https://github.com/mozilla/kuma/blob/71816ee9280238aa5460bd3f7e12eaad9589abb5/kuma/wiki/search.py#L30-L31
But [the way it does it](https://github.com/mozilla/kuma/blob/71816ee9280238aa5460bd3f7e12eaad9589abb5/kuma/wiki/search.py#L228) is that it uses `icontains` instead of `startswith` which is more inefficient and it's also potentially excluding slugs that contain any of those strings later. E.g. `Learn/HTML/user:prefixing`

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `kuma/wiki/search.py`
Content:
```
1 # -*- coding: utf-8 -*-
2 from __future__ import division
3 
4 import logging
5 import operator
6 from math import ceil
7 
8 from celery import chain
9 from django.conf import settings
10 from django.db.models import Q
11 from django.utils.html import strip_tags
12 from django.utils.translation import ugettext_lazy as _
13 from elasticsearch.helpers import bulk
14 from elasticsearch_dsl import document, field
15 from elasticsearch_dsl.connections import connections
16 from elasticsearch_dsl.mapping import Mapping
17 from elasticsearch_dsl.search import Search
18 from six.moves import reduce
19 
20 from kuma.core.utils import chord_flow, chunked
21 
22 from .constants import EXPERIMENT_TITLE_PREFIX
23 
24 
25 log = logging.getLogger('kuma.wiki.search')
26 
27 
28 class WikiDocumentType(document.Document):
29     excerpt_fields = ['summary', 'content']
30     exclude_slugs = ['Talk:', 'User:', 'User_talk:', 'Template_talk:',
31                      'Project_talk:', EXPERIMENT_TITLE_PREFIX]
32 
33     boost = field.Float(null_value=1.0)
34     content = field.Text(analyzer='kuma_content',
35                          term_vector='with_positions_offsets')
36     css_classnames = field.Keyword()
37     html_attributes = field.Keyword()
38     id = field.Long()
39     kumascript_macros = field.Keyword()
40     locale = field.Keyword()
41     modified = field.Date()
42     parent = field.Object(properties={
43         'id': field.Long(),
44         'title': field.Text(analyzer='kuma_title'),
45         'slug': field.Keyword(),
46         'locale': field.Keyword(),
47     })
48     slug = field.Keyword()
49     summary = field.Text(analyzer='kuma_content',
50                          term_vector='with_positions_offsets')
51     tags = field.Keyword()
52     title = field.Text(analyzer='kuma_title')
53 
54     class Meta(object):
55         mapping = Mapping('wiki_document')
56         mapping.meta('_all', enabled=False)
57 
58     @classmethod
59     def get_connection(cls, alias='default'):
60         return connections.get_connection(alias)
61 
62     @classmethod
63     def get_doc_type(cls):
64         return cls._doc_type.name
65 
66     @classmethod
67     def case_insensitive_keywords(cls, keywords):
68         '''Create a unique list of lowercased keywords.'''
69         return sorted(set([keyword.lower() for keyword in keywords]))
70 
71     @classmethod
72     def from_django(cls, obj):
73         is_root_document = obj.slug.count('/') == 1
74         doc = {
75             'id': obj.id,
76             'boost': 4.0 if is_root_document else 1.0,
77             'title': obj.title,
78             'slug': obj.slug,
79             'summary': obj.get_summary_text(),
80             'locale': obj.locale,
81             'modified': obj.modified,
82             'content': strip_tags(obj.rendered_html or ''),
83             'tags': [o.name for o in obj.tags.all()],
84             'kumascript_macros': cls.case_insensitive_keywords(
85                 obj.extract.macro_names()),
86             'css_classnames': cls.case_insensitive_keywords(
87                 obj.extract.css_classnames()),
88             'html_attributes': cls.case_insensitive_keywords(
89                 obj.extract.html_attributes()),
90         }
91 
92         if obj.parent:
93             doc['parent'] = {
94                 'id': obj.parent.id,
95                 'title': obj.parent.title,
96                 'locale': obj.parent.locale,
97                 'slug': obj.parent.slug,
98             }
99         else:
100             doc['parent'] = {}
101 
102         return doc
103 
104     @classmethod
105     def get_mapping(cls):
106         return cls._doc_type.mapping.to_dict()
107 
108     @classmethod
109     def get_analysis(cls):
110         return {
111             'filter': {
112                 'kuma_word_delimiter': {
113                     'type': 'word_delimiter',
114                     'preserve_original': True,  # hi-fi -> hifi, hi-fi
115                     'catenate_words': True,  # hi-fi -> hifi
116                     'catenate_numbers': True,  # 90-210 -> 90210
117                 }
118             },
119             'analyzer': {
120                 'default': {
121                     'tokenizer': 'standard',
122                     'filter': ['standard', 'elision']
123                 },
124                 # a custom analyzer that strips html and uses our own
125                 # word delimiter filter and the elision filter
126                 # (e.g. L'attribut -> attribut). The rest is the same as
127                 # the snowball analyzer
128                 'kuma_content': {
129                     'type': 'custom',
130                     'tokenizer': 'standard',
131                     'char_filter': ['html_strip'],
132                     'filter': [
133                         'elision',
134                         'kuma_word_delimiter',
135                         'lowercase',
136                         'standard',
137                         'stop',
138                         'snowball',
139                     ],
140                 },
141                 'kuma_title': {
142                     'type': 'custom',
143                     'tokenizer': 'standard',
144                     'filter': [
145                         'elision',
146                         'kuma_word_delimiter',
147                         'lowercase',
148                         'standard',
149                         'snowball',
150                     ],
151                 },
152             },
153         }
154 
155     @classmethod
156     def get_settings(cls):
157         return {
158             'mappings': cls.get_mapping(),
159             'settings': {
160                 'analysis': cls.get_analysis(),
161                 'number_of_replicas': settings.ES_DEFAULT_NUM_REPLICAS,
162                 'number_of_shards': settings.ES_DEFAULT_NUM_SHARDS,
163             }
164         }
165 
166     @classmethod
167     def bulk_index(cls, documents, id_field='id', es=None, index=None):
168         """Index of a bunch of documents."""
169         es = es or cls.get_connection()
170         index = index or cls.get_index()
171         type = cls.get_doc_type()
172 
173         actions = [
174             {'_index': index, '_type': type, '_id': d['id'], '_source': d}
175             for d in documents]
176 
177         bulk(es, actions)
178 
179     @classmethod
180     def bulk_delete(cls, ids, es=None, index=None):
181         """Index of a bunch of documents."""
182         es = es or cls.get_connection()
183         index = index or cls.get_index()
184         type = cls.get_doc_type()
185 
186         actions = [
187             {'_op_type': 'delete', '_index': index, '_type': type, '_id': _id}
188             for _id in ids]
189 
190         bulk(es, actions)
191 
192     @classmethod
193     def get_index(cls):
194         from kuma.search.models import Index
195         return Index.objects.get_current().prefixed_name
196 
197     @classmethod
198     def search(cls, **kwargs):
199         options = {
200             'using': connections.get_connection(),
201             'index': cls.get_index(),
202             'doc_type': {cls._doc_type.name: cls.from_es},
203         }
204         options.update(kwargs)
205         sq = Search(**options)
206 
207         return sq
208 
209     @classmethod
210     def get_model(cls):
211         from kuma.wiki.models import Document
212         return Document
213 
214     @classmethod
215     def get_indexable(cls, percent=100):
216         """
217         For this mapping type return a list of model IDs that should be
218         indexed with the management command, in a full reindex.
219 
220         WARNING: When changing this code make sure to update the
221                  ``should_update`` method below, too!
222 
223         """
224         model = cls.get_model()
225 
226         excludes = []
227         for exclude in cls.exclude_slugs:
228             excludes.append(Q(slug__icontains=exclude))
229 
230         qs = (model.objects
231                    .filter(is_redirect=False, deleted=False)
232                    .exclude(reduce(operator.or_, excludes)))
233 
234         percent = percent / 100
235         if percent < 1:
236             qs = qs[:int(qs.count() * percent)]
237 
238         return qs.values_list('id', flat=True)
239 
240     @classmethod
241     def should_update(cls, obj):
242         """
243         Given a Document instance should return boolean value
244         whether the instance should be indexed or not.
245 
246         WARNING: This *must* mirror the logic of the ``get_indexable``
247                  method above!
248         """
249         return (not obj.is_redirect and not obj.deleted and
250                 not any([exclude in obj.slug
251                          for exclude in cls.exclude_slugs]))
252 
253     def get_excerpt(self):
254         highlighted = getattr(self.meta, 'highlight', None)
255         if highlighted:
256             for excerpt_field in self.excerpt_fields:
257                 if excerpt_field in highlighted:
258                     return u'…'.join(highlighted[excerpt_field])
259         return self.summary
260 
261     @classmethod
262     def reindex_all(cls, chunk_size=500, index=None, percent=100):
263         """Rebuild ElasticSearch indexes.
264 
265         :arg chunk_size: how many documents to bulk index as a single chunk.
266         :arg index: the `Index` object to reindex into. Uses the current
267             promoted index if none provided.
268         :arg percent: 1 to 100--the percentage of the db to index.
269 
270         """
271         from kuma.search.models import Index
272         from kuma.search.tasks import prepare_index, finalize_index
273         from kuma.wiki.tasks import index_documents
274 
275         index = index or Index.objects.get_current()
276 
277         # Get the list of document IDs to index.
278         indexable = WikiDocumentType.get_indexable(percent)
279 
280         total = len(indexable)
281         total_chunks = int(ceil(total / chunk_size))
282 
283         pre_task = prepare_index.si(index.pk)
284         post_task = finalize_index.si(index.pk)
285 
286         if not total:
287             # If there's no data we still create the index and finalize it.
288             chain(pre_task, post_task).apply_async()
289         else:
290             index_tasks = [index_documents.si(chunk, index.pk)
291                            for chunk in chunked(indexable, chunk_size)]
292             chord_flow(pre_task, index_tasks, post_task).apply_async()
293 
294         message = _(
295             'Indexing %(total)d documents into %(total_chunks)d chunks of '
296             'size %(size)d into index %(index)s.' % {
297                 'total': total,
298                 'total_chunks': total_chunks,
299                 'size': chunk_size,
300                 'index': index.prefixed_name
301             }
302         )
303         return message
304
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/kuma/wiki/search.py b/kuma/wiki/search.py
--- a/kuma/wiki/search.py
+++ b/kuma/wiki/search.py
@@ -2,7 +2,6 @@
 from __future__ import division
 
 import logging
-import operator
 from math import ceil
 
 from celery import chain
@@ -15,7 +14,6 @@
 from elasticsearch_dsl.connections import connections
 from elasticsearch_dsl.mapping import Mapping
 from elasticsearch_dsl.search import Search
-from six.moves import reduce
 
 from kuma.core.utils import chord_flow, chunked
 
@@ -223,13 +221,11 @@
         """
         model = cls.get_model()
 
-        excludes = []
+        excludes = Q()
         for exclude in cls.exclude_slugs:
-            excludes.append(Q(slug__icontains=exclude))
+            excludes |= Q(slug__startswith=exclude)
 
-        qs = (model.objects
-                   .filter(is_redirect=False, deleted=False)
-                   .exclude(reduce(operator.or_, excludes)))
+        qs = model.objects.filter(is_redirect=False).exclude(excludes)
 
         percent = percent / 100
         if percent < 1:

{"golden_diff": "diff --git a/kuma/wiki/search.py b/kuma/wiki/search.py\n--- a/kuma/wiki/search.py\n+++ b/kuma/wiki/search.py\n@@ -2,7 +2,6 @@\n from __future__ import division\n \n import logging\n-import operator\n from math import ceil\n \n from celery import chain\n@@ -15,7 +14,6 @@\n from elasticsearch_dsl.connections import connections\n from elasticsearch_dsl.mapping import Mapping\n from elasticsearch_dsl.search import Search\n-from six.moves import reduce\n \n from kuma.core.utils import chord_flow, chunked\n \n@@ -223,13 +221,11 @@\n         \"\"\"\n         model = cls.get_model()\n \n-        excludes = []\n+        excludes = Q()\n         for exclude in cls.exclude_slugs:\n-            excludes.append(Q(slug__icontains=exclude))\n+            excludes |= Q(slug__startswith=exclude)\n \n-        qs = (model.objects\n-                   .filter(is_redirect=False, deleted=False)\n-                   .exclude(reduce(operator.or_, excludes)))\n+        qs = model.objects.filter(is_redirect=False).exclude(excludes)\n \n         percent = percent / 100\n         if percent < 1:\n", "issue": "Reindex dangerously excludes slugs\nThe reindexing exludes these slugs: https://github.com/mozilla/kuma/blob/71816ee9280238aa5460bd3f7e12eaad9589abb5/kuma/wiki/search.py#L30-L31\r\nBut [the way it does it](https://github.com/mozilla/kuma/blob/71816ee9280238aa5460bd3f7e12eaad9589abb5/kuma/wiki/search.py#L228) is that it uses `icontains` instead of `startswith` which is more inefficient and it's also potentially excluding slugs that contain any of those strings later. E.g. `Learn/HTML/user:prefixing`\n", "before_files": [{"content": "# -*- coding: utf-8 -*-\nfrom __future__ import division\n\nimport logging\nimport operator\nfrom math import ceil\n\nfrom celery import chain\nfrom django.conf import settings\nfrom django.db.models import Q\nfrom django.utils.html import strip_tags\nfrom django.utils.translation import ugettext_lazy as _\nfrom elasticsearch.helpers import bulk\nfrom elasticsearch_dsl import document, field\nfrom elasticsearch_dsl.connections import connections\nfrom elasticsearch_dsl.mapping import Mapping\nfrom elasticsearch_dsl.search import Search\nfrom six.moves import reduce\n\nfrom kuma.core.utils import chord_flow, chunked\n\nfrom .constants import EXPERIMENT_TITLE_PREFIX\n\n\nlog = logging.getLogger('kuma.wiki.search')\n\n\nclass WikiDocumentType(document.Document):\n    excerpt_fields = ['summary', 'content']\n    exclude_slugs = ['Talk:', 'User:', 'User_talk:', 'Template_talk:',\n                     'Project_talk:', EXPERIMENT_TITLE_PREFIX]\n\n    boost = field.Float(null_value=1.0)\n    content = field.Text(analyzer='kuma_content',\n                         term_vector='with_positions_offsets')\n    css_classnames = field.Keyword()\n    html_attributes = field.Keyword()\n    id = field.Long()\n    kumascript_macros = field.Keyword()\n    locale = field.Keyword()\n    modified = field.Date()\n    parent = field.Object(properties={\n        'id': field.Long(),\n        'title': field.Text(analyzer='kuma_title'),\n        'slug': field.Keyword(),\n        'locale': field.Keyword(),\n    })\n    slug = field.Keyword()\n    summary = field.Text(analyzer='kuma_content',\n                         term_vector='with_positions_offsets')\n    tags = field.Keyword()\n    title = field.Text(analyzer='kuma_title')\n\n    class Meta(object):\n        mapping = Mapping('wiki_document')\n        mapping.meta('_all', enabled=False)\n\n    @classmethod\n    def get_connection(cls, alias='default'):\n        return connections.get_connection(alias)\n\n    @classmethod\n    def get_doc_type(cls):\n        return cls._doc_type.name\n\n    @classmethod\n    def case_insensitive_keywords(cls, keywords):\n        '''Create a unique list of lowercased keywords.'''\n        return sorted(set([keyword.lower() for keyword in keywords]))\n\n    @classmethod\n    def from_django(cls, obj):\n        is_root_document = obj.slug.count('/') == 1\n        doc = {\n            'id': obj.id,\n            'boost': 4.0 if is_root_document else 1.0,\n            'title': obj.title,\n            'slug': obj.slug,\n            'summary': obj.get_summary_text(),\n            'locale': obj.locale,\n            'modified': obj.modified,\n            'content': strip_tags(obj.rendered_html or ''),\n            'tags': [o.name for o in obj.tags.all()],\n            'kumascript_macros': cls.case_insensitive_keywords(\n                obj.extract.macro_names()),\n            'css_classnames': cls.case_insensitive_keywords(\n                obj.extract.css_classnames()),\n            'html_attributes': cls.case_insensitive_keywords(\n                obj.extract.html_attributes()),\n        }\n\n        if obj.parent:\n            doc['parent'] = {\n                'id': obj.parent.id,\n                'title': obj.parent.title,\n                'locale': obj.parent.locale,\n                'slug': obj.parent.slug,\n            }\n        else:\n            doc['parent'] = {}\n\n        return doc\n\n    @classmethod\n    def get_mapping(cls):\n        return cls._doc_type.mapping.to_dict()\n\n    @classmethod\n    def get_analysis(cls):\n        return {\n            'filter': {\n                'kuma_word_delimiter': {\n                    'type': 'word_delimiter',\n                    'preserve_original': True,  # hi-fi -> hifi, hi-fi\n                    'catenate_words': True,  # hi-fi -> hifi\n                    'catenate_numbers': True,  # 90-210 -> 90210\n                }\n            },\n            'analyzer': {\n                'default': {\n                    'tokenizer': 'standard',\n                    'filter': ['standard', 'elision']\n                },\n                # a custom analyzer that strips html and uses our own\n                # word delimiter filter and the elision filter\n                # (e.g. L'attribut -> attribut). The rest is the same as\n                # the snowball analyzer\n                'kuma_content': {\n                    'type': 'custom',\n                    'tokenizer': 'standard',\n                    'char_filter': ['html_strip'],\n                    'filter': [\n                        'elision',\n                        'kuma_word_delimiter',\n                        'lowercase',\n                        'standard',\n                        'stop',\n                        'snowball',\n                    ],\n                },\n                'kuma_title': {\n                    'type': 'custom',\n                    'tokenizer': 'standard',\n                    'filter': [\n                        'elision',\n                        'kuma_word_delimiter',\n                        'lowercase',\n                        'standard',\n                        'snowball',\n                    ],\n                },\n            },\n        }\n\n    @classmethod\n    def get_settings(cls):\n        return {\n            'mappings': cls.get_mapping(),\n            'settings': {\n                'analysis': cls.get_analysis(),\n                'number_of_replicas': settings.ES_DEFAULT_NUM_REPLICAS,\n                'number_of_shards': settings.ES_DEFAULT_NUM_SHARDS,\n            }\n        }\n\n    @classmethod\n    def bulk_index(cls, documents, id_field='id', es=None, index=None):\n        \"\"\"Index of a bunch of documents.\"\"\"\n        es = es or cls.get_connection()\n        index = index or cls.get_index()\n        type = cls.get_doc_type()\n\n        actions = [\n            {'_index': index, '_type': type, '_id': d['id'], '_source': d}\n            for d in documents]\n\n        bulk(es, actions)\n\n    @classmethod\n    def bulk_delete(cls, ids, es=None, index=None):\n        \"\"\"Index of a bunch of documents.\"\"\"\n        es = es or cls.get_connection()\n        index = index or cls.get_index()\n        type = cls.get_doc_type()\n\n        actions = [\n            {'_op_type': 'delete', '_index': index, '_type': type, '_id': _id}\n            for _id in ids]\n\n        bulk(es, actions)\n\n    @classmethod\n    def get_index(cls):\n        from kuma.search.models import Index\n        return Index.objects.get_current().prefixed_name\n\n    @classmethod\n    def search(cls, **kwargs):\n        options = {\n            'using': connections.get_connection(),\n            'index': cls.get_index(),\n            'doc_type': {cls._doc_type.name: cls.from_es},\n        }\n        options.update(kwargs)\n        sq = Search(**options)\n\n        return sq\n\n    @classmethod\n    def get_model(cls):\n        from kuma.wiki.models import Document\n        return Document\n\n    @classmethod\n    def get_indexable(cls, percent=100):\n        \"\"\"\n        For this mapping type return a list of model IDs that should be\n        indexed with the management command, in a full reindex.\n\n        WARNING: When changing this code make sure to update the\n                 ``should_update`` method below, too!\n\n        \"\"\"\n        model = cls.get_model()\n\n        excludes = []\n        for exclude in cls.exclude_slugs:\n            excludes.append(Q(slug__icontains=exclude))\n\n        qs = (model.objects\n                   .filter(is_redirect=False, deleted=False)\n                   .exclude(reduce(operator.or_, excludes)))\n\n        percent = percent / 100\n        if percent < 1:\n            qs = qs[:int(qs.count() * percent)]\n\n        return qs.values_list('id', flat=True)\n\n    @classmethod\n    def should_update(cls, obj):\n        \"\"\"\n        Given a Document instance should return boolean value\n        whether the instance should be indexed or not.\n\n        WARNING: This *must* mirror the logic of the ``get_indexable``\n                 method above!\n        \"\"\"\n        return (not obj.is_redirect and not obj.deleted and\n                not any([exclude in obj.slug\n                         for exclude in cls.exclude_slugs]))\n\n    def get_excerpt(self):\n        highlighted = getattr(self.meta, 'highlight', None)\n        if highlighted:\n            for excerpt_field in self.excerpt_fields:\n                if excerpt_field in highlighted:\n                    return u'\u2026'.join(highlighted[excerpt_field])\n        return self.summary\n\n    @classmethod\n    def reindex_all(cls, chunk_size=500, index=None, percent=100):\n        \"\"\"Rebuild ElasticSearch indexes.\n\n        :arg chunk_size: how many documents to bulk index as a single chunk.\n        :arg index: the `Index` object to reindex into. Uses the current\n            promoted index if none provided.\n        :arg percent: 1 to 100--the percentage of the db to index.\n\n        \"\"\"\n        from kuma.search.models import Index\n        from kuma.search.tasks import prepare_index, finalize_index\n        from kuma.wiki.tasks import index_documents\n\n        index = index or Index.objects.get_current()\n\n        # Get the list of document IDs to index.\n        indexable = WikiDocumentType.get_indexable(percent)\n\n        total = len(indexable)\n        total_chunks = int(ceil(total / chunk_size))\n\n        pre_task = prepare_index.si(index.pk)\n        post_task = finalize_index.si(index.pk)\n\n        if not total:\n            # If there's no data we still create the index and finalize it.\n            chain(pre_task, post_task).apply_async()\n        else:\n            index_tasks = [index_documents.si(chunk, index.pk)\n                           for chunk in chunked(indexable, chunk_size)]\n            chord_flow(pre_task, index_tasks, post_task).apply_async()\n\n        message = _(\n            'Indexing %(total)d documents into %(total_chunks)d chunks of '\n            'size %(size)d into index %(index)s.' % {\n                'total': total,\n                'total_chunks': total_chunks,\n                'size': chunk_size,\n                'index': index.prefixed_name\n            }\n        )\n        return message\n", "path": "kuma/wiki/search.py"}], "after_files": [{"content": "# -*- coding: utf-8 -*-\nfrom __future__ import division\n\nimport logging\nfrom math import ceil\n\nfrom celery import chain\nfrom django.conf import settings\nfrom django.db.models import Q\nfrom django.utils.html import strip_tags\nfrom django.utils.translation import ugettext_lazy as _\nfrom elasticsearch.helpers import bulk\nfrom elasticsearch_dsl import document, field\nfrom elasticsearch_dsl.connections import connections\nfrom elasticsearch_dsl.mapping import Mapping\nfrom elasticsearch_dsl.search import Search\n\nfrom kuma.core.utils import chord_flow, chunked\n\nfrom .constants import EXPERIMENT_TITLE_PREFIX\n\n\nlog = logging.getLogger('kuma.wiki.search')\n\n\nclass WikiDocumentType(document.Document):\n    excerpt_fields = ['summary', 'content']\n    exclude_slugs = ['Talk:', 'User:', 'User_talk:', 'Template_talk:',\n                     'Project_talk:', EXPERIMENT_TITLE_PREFIX]\n\n    boost = field.Float(null_value=1.0)\n    content = field.Text(analyzer='kuma_content',\n                         term_vector='with_positions_offsets')\n    css_classnames = field.Keyword()\n    html_attributes = field.Keyword()\n    id = field.Long()\n    kumascript_macros = field.Keyword()\n    locale = field.Keyword()\n    modified = field.Date()\n    parent = field.Object(properties={\n        'id': field.Long(),\n        'title': field.Text(analyzer='kuma_title'),\n        'slug': field.Keyword(),\n        'locale': field.Keyword(),\n    })\n    slug = field.Keyword()\n    summary = field.Text(analyzer='kuma_content',\n                         term_vector='with_positions_offsets')\n    tags = field.Keyword()\n    title = field.Text(analyzer='kuma_title')\n\n    class Meta(object):\n        mapping = Mapping('wiki_document')\n        mapping.meta('_all', enabled=False)\n\n    @classmethod\n    def get_connection(cls, alias='default'):\n        return connections.get_connection(alias)\n\n    @classmethod\n    def get_doc_type(cls):\n        return cls._doc_type.name\n\n    @classmethod\n    def case_insensitive_keywords(cls, keywords):\n        '''Create a unique list of lowercased keywords.'''\n        return sorted(set([keyword.lower() for keyword in keywords]))\n\n    @classmethod\n    def from_django(cls, obj):\n        is_root_document = obj.slug.count('/') == 1\n        doc = {\n            'id': obj.id,\n            'boost': 4.0 if is_root_document else 1.0,\n            'title': obj.title,\n            'slug': obj.slug,\n            'summary': obj.get_summary_text(),\n            'locale': obj.locale,\n            'modified': obj.modified,\n            'content': strip_tags(obj.rendered_html or ''),\n            'tags': [o.name for o in obj.tags.all()],\n            'kumascript_macros': cls.case_insensitive_keywords(\n                obj.extract.macro_names()),\n            'css_classnames': cls.case_insensitive_keywords(\n                obj.extract.css_classnames()),\n            'html_attributes': cls.case_insensitive_keywords(\n                obj.extract.html_attributes()),\n        }\n\n        if obj.parent:\n            doc['parent'] = {\n                'id': obj.parent.id,\n                'title': obj.parent.title,\n                'locale': obj.parent.locale,\n                'slug': obj.parent.slug,\n            }\n        else:\n            doc['parent'] = {}\n\n        return doc\n\n    @classmethod\n    def get_mapping(cls):\n        return cls._doc_type.mapping.to_dict()\n\n    @classmethod\n    def get_analysis(cls):\n        return {\n            'filter': {\n                'kuma_word_delimiter': {\n                    'type': 'word_delimiter',\n                    'preserve_original': True,  # hi-fi -> hifi, hi-fi\n                    'catenate_words': True,  # hi-fi -> hifi\n                    'catenate_numbers': True,  # 90-210 -> 90210\n                }\n            },\n            'analyzer': {\n                'default': {\n                    'tokenizer': 'standard',\n                    'filter': ['standard', 'elision']\n                },\n                # a custom analyzer that strips html and uses our own\n                # word delimiter filter and the elision filter\n                # (e.g. L'attribut -> attribut). The rest is the same as\n                # the snowball analyzer\n                'kuma_content': {\n                    'type': 'custom',\n                    'tokenizer': 'standard',\n                    'char_filter': ['html_strip'],\n                    'filter': [\n                        'elision',\n                        'kuma_word_delimiter',\n                        'lowercase',\n                        'standard',\n                        'stop',\n                        'snowball',\n                    ],\n                },\n                'kuma_title': {\n                    'type': 'custom',\n                    'tokenizer': 'standard',\n                    'filter': [\n                        'elision',\n                        'kuma_word_delimiter',\n                        'lowercase',\n                        'standard',\n                        'snowball',\n                    ],\n                },\n            },\n        }\n\n    @classmethod\n    def get_settings(cls):\n        return {\n            'mappings': cls.get_mapping(),\n            'settings': {\n                'analysis': cls.get_analysis(),\n                'number_of_replicas': settings.ES_DEFAULT_NUM_REPLICAS,\n                'number_of_shards': settings.ES_DEFAULT_NUM_SHARDS,\n            }\n        }\n\n    @classmethod\n    def bulk_index(cls, documents, id_field='id', es=None, index=None):\n        \"\"\"Index of a bunch of documents.\"\"\"\n        es = es or cls.get_connection()\n        index = index or cls.get_index()\n        type = cls.get_doc_type()\n\n        actions = [\n            {'_index': index, '_type': type, '_id': d['id'], '_source': d}\n            for d in documents]\n\n        bulk(es, actions)\n\n    @classmethod\n    def bulk_delete(cls, ids, es=None, index=None):\n        \"\"\"Index of a bunch of documents.\"\"\"\n        es = es or cls.get_connection()\n        index = index or cls.get_index()\n        type = cls.get_doc_type()\n\n        actions = [\n            {'_op_type': 'delete', '_index': index, '_type': type, '_id': _id}\n            for _id in ids]\n\n        bulk(es, actions)\n\n    @classmethod\n    def get_index(cls):\n        from kuma.search.models import Index\n        return Index.objects.get_current().prefixed_name\n\n    @classmethod\n    def search(cls, **kwargs):\n        options = {\n            'using': connections.get_connection(),\n            'index': cls.get_index(),\n            'doc_type': {cls._doc_type.name: cls.from_es},\n        }\n        options.update(kwargs)\n        sq = Search(**options)\n\n        return sq\n\n    @classmethod\n    def get_model(cls):\n        from kuma.wiki.models import Document\n        return Document\n\n    @classmethod\n    def get_indexable(cls, percent=100):\n        \"\"\"\n        For this mapping type return a list of model IDs that should be\n        indexed with the management command, in a full reindex.\n\n        WARNING: When changing this code make sure to update the\n                 ``should_update`` method below, too!\n\n        \"\"\"\n        model = cls.get_model()\n\n        excludes = Q()\n        for exclude in cls.exclude_slugs:\n            excludes |= Q(slug__startswith=exclude)\n\n        qs = model.objects.filter(is_redirect=False).exclude(excludes)\n\n        percent = percent / 100\n        if percent < 1:\n            qs = qs[:int(qs.count() * percent)]\n\n        return qs.values_list('id', flat=True)\n\n    @classmethod\n    def should_update(cls, obj):\n        \"\"\"\n        Given a Document instance should return boolean value\n        whether the instance should be indexed or not.\n\n        WARNING: This *must* mirror the logic of the ``get_indexable``\n                 method above!\n        \"\"\"\n        return (not obj.is_redirect and not obj.deleted and\n                not any([exclude in obj.slug\n                         for exclude in cls.exclude_slugs]))\n\n    def get_excerpt(self):\n        highlighted = getattr(self.meta, 'highlight', None)\n        if highlighted:\n            for excerpt_field in self.excerpt_fields:\n                if excerpt_field in highlighted:\n                    return u'\u2026'.join(highlighted[excerpt_field])\n        return self.summary\n\n    @classmethod\n    def reindex_all(cls, chunk_size=500, index=None, percent=100):\n        \"\"\"Rebuild ElasticSearch indexes.\n\n        :arg chunk_size: how many documents to bulk index as a single chunk.\n        :arg index: the `Index` object to reindex into. Uses the current\n            promoted index if none provided.\n        :arg percent: 1 to 100--the percentage of the db to index.\n\n        \"\"\"\n        from kuma.search.models import Index\n        from kuma.search.tasks import prepare_index, finalize_index\n        from kuma.wiki.tasks import index_documents\n\n        index = index or Index.objects.get_current()\n\n        # Get the list of document IDs to index.\n        indexable = WikiDocumentType.get_indexable(percent)\n\n        total = len(indexable)\n        total_chunks = int(ceil(total / chunk_size))\n\n        pre_task = prepare_index.si(index.pk)\n        post_task = finalize_index.si(index.pk)\n\n        if not total:\n            # If there's no data we still create the index and finalize it.\n            chain(pre_task, post_task).apply_async()\n        else:\n            index_tasks = [index_documents.si(chunk, index.pk)\n                           for chunk in chunked(indexable, chunk_size)]\n            chord_flow(pre_task, index_tasks, post_task).apply_async()\n\n        message = _(\n            'Indexing %(total)d documents into %(total_chunks)d chunks of '\n            'size %(size)d into index %(index)s.' % {\n                'total': total,\n                'total_chunks': total_chunks,\n                'size': chunk_size,\n                'index': index.prefixed_name\n            }\n        )\n        return message\n", "path": "kuma/wiki/search.py"}]}