We are currently solving the following issue within our repository. Here is the issue text:
--- BEGIN ISSUE ---
select_related() ConnectionError 
Hi,

I am having an interesting situation. We have been using MongoEngine in our project for a good few years now and this issue appeared out of nowhere (no update, no change to the code).

Calling `select_related()` on the results of a query raises a `mongoengine.connection.ConnectionError: You have not defined a default connection` Exception

The call that causes the issue:
```credentials = Credential.objects(id__in=ids).select_related()```

I noticed that I am getting the exception only the first time time I call `select_related()` and only on this object. Any subsequent calls (on the same collection) work as expected. 

While trying to debug, I got to this:
```
> /usr/local/lib/python2.7/dist-packages/mongoengine/dereference.py(129)_fetch_objects()
    127         object_map = {}
    128         for collection, dbrefs in self.reference_map.iteritems():
--> 129             if hasattr(collection, 'objects'):  # We have a document class for the refs
    130                 col_name = collection._get_collection_name()
    131                 refs = [dbref for dbref in dbrefs

ipdb> collection
<class 'insights.models.company.Company'>
ipdb> hasattr(collection, 'objects')
False
ipdb> hasattr(collection, 'objects')
True
```

Can this be caused by some lazy loading or some configuration issue?

Thank you!

--- END ISSUE ---

Below are some code segments, each from a relevant file. One or more of these files may contain bugs.
--- BEGIN FILES ---
Path: `mongoengine/dereference.py`
Content:
```
1 from bson import DBRef, SON
2 import six
3 
4 from mongoengine.base import (BaseDict, BaseList, EmbeddedDocumentList,
5                               TopLevelDocumentMetaclass, get_document)
6 from mongoengine.base.datastructures import LazyReference
7 from mongoengine.connection import get_db
8 from mongoengine.document import Document, EmbeddedDocument
9 from mongoengine.fields import DictField, ListField, MapField, ReferenceField
10 from mongoengine.queryset import QuerySet
11 
12 
13 class DeReference(object):
14     def __call__(self, items, max_depth=1, instance=None, name=None):
15         """
16         Cheaply dereferences the items to a set depth.
17         Also handles the conversion of complex data types.
18 
19         :param items: The iterable (dict, list, queryset) to be dereferenced.
20         :param max_depth: The maximum depth to recurse to
21         :param instance: The owning instance used for tracking changes by
22             :class:`~mongoengine.base.ComplexBaseField`
23         :param name: The name of the field, used for tracking changes by
24             :class:`~mongoengine.base.ComplexBaseField`
25         :param get: A boolean determining if being called by __get__
26         """
27         if items is None or isinstance(items, six.string_types):
28             return items
29 
30         # cheapest way to convert a queryset to a list
31         # list(queryset) uses a count() query to determine length
32         if isinstance(items, QuerySet):
33             items = [i for i in items]
34 
35         self.max_depth = max_depth
36         doc_type = None
37 
38         if instance and isinstance(instance, (Document, EmbeddedDocument,
39                                               TopLevelDocumentMetaclass)):
40             doc_type = instance._fields.get(name)
41             while hasattr(doc_type, 'field'):
42                 doc_type = doc_type.field
43 
44             if isinstance(doc_type, ReferenceField):
45                 field = doc_type
46                 doc_type = doc_type.document_type
47                 is_list = not hasattr(items, 'items')
48 
49                 if is_list and all([i.__class__ == doc_type for i in items]):
50                     return items
51                 elif not is_list and all(
52                         [i.__class__ == doc_type for i in items.values()]):
53                     return items
54                 elif not field.dbref:
55                     if not hasattr(items, 'items'):
56 
57                         def _get_items(items):
58                             new_items = []
59                             for v in items:
60                                 if isinstance(v, list):
61                                     new_items.append(_get_items(v))
62                                 elif not isinstance(v, (DBRef, Document)):
63                                     new_items.append(field.to_python(v))
64                                 else:
65                                     new_items.append(v)
66                             return new_items
67 
68                         items = _get_items(items)
69                     else:
70                         items = {
71                             k: (v if isinstance(v, (DBRef, Document))
72                                 else field.to_python(v))
73                             for k, v in items.iteritems()
74                         }
75 
76         self.reference_map = self._find_references(items)
77         self.object_map = self._fetch_objects(doc_type=doc_type)
78         return self._attach_objects(items, 0, instance, name)
79 
80     def _find_references(self, items, depth=0):
81         """
82         Recursively finds all db references to be dereferenced
83 
84         :param items: The iterable (dict, list, queryset)
85         :param depth: The current depth of recursion
86         """
87         reference_map = {}
88         if not items or depth >= self.max_depth:
89             return reference_map
90 
91         # Determine the iterator to use
92         if isinstance(items, dict):
93             iterator = items.values()
94         else:
95             iterator = items
96 
97         # Recursively find dbreferences
98         depth += 1
99         for item in iterator:
100             if isinstance(item, (Document, EmbeddedDocument)):
101                 for field_name, field in item._fields.iteritems():
102                     v = item._data.get(field_name, None)
103                     if isinstance(v, LazyReference):
104                         # LazyReference inherits DBRef but should not be dereferenced here !
105                         continue
106                     elif isinstance(v, DBRef):
107                         reference_map.setdefault(field.document_type, set()).add(v.id)
108                     elif isinstance(v, (dict, SON)) and '_ref' in v:
109                         reference_map.setdefault(get_document(v['_cls']), set()).add(v['_ref'].id)
110                     elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth:
111                         field_cls = getattr(getattr(field, 'field', None), 'document_type', None)
112                         references = self._find_references(v, depth)
113                         for key, refs in references.iteritems():
114                             if isinstance(field_cls, (Document, TopLevelDocumentMetaclass)):
115                                 key = field_cls
116                             reference_map.setdefault(key, set()).update(refs)
117             elif isinstance(item, LazyReference):
118                 # LazyReference inherits DBRef but should not be dereferenced here !
119                 continue
120             elif isinstance(item, DBRef):
121                 reference_map.setdefault(item.collection, set()).add(item.id)
122             elif isinstance(item, (dict, SON)) and '_ref' in item:
123                 reference_map.setdefault(get_document(item['_cls']), set()).add(item['_ref'].id)
124             elif isinstance(item, (dict, list, tuple)) and depth - 1 <= self.max_depth:
125                 references = self._find_references(item, depth - 1)
126                 for key, refs in references.iteritems():
127                     reference_map.setdefault(key, set()).update(refs)
128 
129         return reference_map
130 
131     def _fetch_objects(self, doc_type=None):
132         """Fetch all references and convert to their document objects
133         """
134         object_map = {}
135         for collection, dbrefs in self.reference_map.iteritems():
136             if hasattr(collection, 'objects'):  # We have a document class for the refs
137                 col_name = collection._get_collection_name()
138                 refs = [dbref for dbref in dbrefs
139                         if (col_name, dbref) not in object_map]
140                 references = collection.objects.in_bulk(refs)
141                 for key, doc in references.iteritems():
142                     object_map[(col_name, key)] = doc
143             else:  # Generic reference: use the refs data to convert to document
144                 if isinstance(doc_type, (ListField, DictField, MapField,)):
145                     continue
146 
147                 refs = [dbref for dbref in dbrefs
148                         if (collection, dbref) not in object_map]
149 
150                 if doc_type:
151                     references = doc_type._get_db()[collection].find({'_id': {'$in': refs}})
152                     for ref in references:
153                         doc = doc_type._from_son(ref)
154                         object_map[(collection, doc.id)] = doc
155                 else:
156                     references = get_db()[collection].find({'_id': {'$in': refs}})
157                     for ref in references:
158                         if '_cls' in ref:
159                             doc = get_document(ref['_cls'])._from_son(ref)
160                         elif doc_type is None:
161                             doc = get_document(
162                                 ''.join(x.capitalize()
163                                         for x in collection.split('_')))._from_son(ref)
164                         else:
165                             doc = doc_type._from_son(ref)
166                         object_map[(collection, doc.id)] = doc
167         return object_map
168 
169     def _attach_objects(self, items, depth=0, instance=None, name=None):
170         """
171         Recursively finds all db references to be dereferenced
172 
173         :param items: The iterable (dict, list, queryset)
174         :param depth: The current depth of recursion
175         :param instance: The owning instance used for tracking changes by
176             :class:`~mongoengine.base.ComplexBaseField`
177         :param name: The name of the field, used for tracking changes by
178             :class:`~mongoengine.base.ComplexBaseField`
179         """
180         if not items:
181             if isinstance(items, (BaseDict, BaseList)):
182                 return items
183 
184             if instance:
185                 if isinstance(items, dict):
186                     return BaseDict(items, instance, name)
187                 else:
188                     return BaseList(items, instance, name)
189 
190         if isinstance(items, (dict, SON)):
191             if '_ref' in items:
192                 return self.object_map.get(
193                     (items['_ref'].collection, items['_ref'].id), items)
194             elif '_cls' in items:
195                 doc = get_document(items['_cls'])._from_son(items)
196                 _cls = doc._data.pop('_cls', None)
197                 del items['_cls']
198                 doc._data = self._attach_objects(doc._data, depth, doc, None)
199                 if _cls is not None:
200                     doc._data['_cls'] = _cls
201                 return doc
202 
203         if not hasattr(items, 'items'):
204             is_list = True
205             list_type = BaseList
206             if isinstance(items, EmbeddedDocumentList):
207                 list_type = EmbeddedDocumentList
208             as_tuple = isinstance(items, tuple)
209             iterator = enumerate(items)
210             data = []
211         else:
212             is_list = False
213             iterator = items.iteritems()
214             data = {}
215 
216         depth += 1
217         for k, v in iterator:
218             if is_list:
219                 data.append(v)
220             else:
221                 data[k] = v
222 
223             if k in self.object_map and not is_list:
224                 data[k] = self.object_map[k]
225             elif isinstance(v, (Document, EmbeddedDocument)):
226                 for field_name in v._fields:
227                     v = data[k]._data.get(field_name, None)
228                     if isinstance(v, DBRef):
229                         data[k]._data[field_name] = self.object_map.get(
230                             (v.collection, v.id), v)
231                     elif isinstance(v, (dict, SON)) and '_ref' in v:
232                         data[k]._data[field_name] = self.object_map.get(
233                             (v['_ref'].collection, v['_ref'].id), v)
234                     elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth:
235                         item_name = six.text_type('{0}.{1}.{2}').format(name, k, field_name)
236                         data[k]._data[field_name] = self._attach_objects(v, depth, instance=instance, name=item_name)
237             elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth:
238                 item_name = '%s.%s' % (name, k) if name else name
239                 data[k] = self._attach_objects(v, depth - 1, instance=instance, name=item_name)
240             elif isinstance(v, DBRef) and hasattr(v, 'id'):
241                 data[k] = self.object_map.get((v.collection, v.id), v)
242 
243         if instance and name:
244             if is_list:
245                 return tuple(data) if as_tuple else list_type(data, instance, name)
246             return BaseDict(data, instance, name)
247         depth += 1
248         return data
249
```
--- END FILES ---

Please first localize the bug based on the issue statement, and then generate a patch according to the `git diff` format fenced by three backticks.

Here is an example:

```diff
diff --git a/examples/server_async.py b/examples/server_async.py
--- a/examples/server_async.py
+++ b/examples/server_async.py
@@ -313,4 +313,4 @@
 
 
 if __name__ == "__main__":
-    asyncio.run(run_async_server("."), debug=True)
+    asyncio.run(run_async_server(), debug=True)
diff --git a/examples/server_sync.py b/examples/server_sync.py
--- a/examples/server_sync.py
+++ b/examples/server_sync.py
@@ -313,5 +313,5 @@
 
 
 if __name__ == "__main__":
-    server = run_sync_server(".")
+    server = run_sync_server()
     server.shutdown()

```

diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py
--- a/mongoengine/dereference.py
+++ b/mongoengine/dereference.py
@@ -133,7 +133,12 @@
         """
         object_map = {}
         for collection, dbrefs in self.reference_map.iteritems():
-            if hasattr(collection, 'objects'):  # We have a document class for the refs
+
+            # we use getattr instead of hasattr because as hasattr swallows any exception under python2
+            # so it could hide nasty things without raising exceptions (cfr bug #1688))
+            ref_document_cls_exists = (getattr(collection, 'objects', None) is not None)
+
+            if ref_document_cls_exists:
                 col_name = collection._get_collection_name()
                 refs = [dbref for dbref in dbrefs
                         if (col_name, dbref) not in object_map]

{"golden_diff": "diff --git a/mongoengine/dereference.py b/mongoengine/dereference.py\n--- a/mongoengine/dereference.py\n+++ b/mongoengine/dereference.py\n@@ -133,7 +133,12 @@\n         \"\"\"\n         object_map = {}\n         for collection, dbrefs in self.reference_map.iteritems():\n-            if hasattr(collection, 'objects'):  # We have a document class for the refs\n+\n+            # we use getattr instead of hasattr because as hasattr swallows any exception under python2\n+            # so it could hide nasty things without raising exceptions (cfr bug #1688))\n+            ref_document_cls_exists = (getattr(collection, 'objects', None) is not None)\n+\n+            if ref_document_cls_exists:\n                 col_name = collection._get_collection_name()\n                 refs = [dbref for dbref in dbrefs\n                         if (col_name, dbref) not in object_map]\n", "issue": "select_related() ConnectionError \nHi,\r\n\r\nI am having an interesting situation. We have been using MongoEngine in our project for a good few years now and this issue appeared out of nowhere (no update, no change to the code).\r\n\r\nCalling `select_related()` on the results of a query raises a `mongoengine.connection.ConnectionError: You have not defined a default connection` Exception\r\n\r\nThe call that causes the issue:\r\n```credentials = Credential.objects(id__in=ids).select_related()```\r\n\r\nI noticed that I am getting the exception only the first time time I call `select_related()` and only on this object. Any subsequent calls (on the same collection) work as expected. \r\n\r\nWhile trying to debug, I got to this:\r\n```\r\n> /usr/local/lib/python2.7/dist-packages/mongoengine/dereference.py(129)_fetch_objects()\r\n    127         object_map = {}\r\n    128         for collection, dbrefs in self.reference_map.iteritems():\r\n--> 129             if hasattr(collection, 'objects'):  # We have a document class for the refs\r\n    130                 col_name = collection._get_collection_name()\r\n    131                 refs = [dbref for dbref in dbrefs\r\n\r\nipdb> collection\r\n<class 'insights.models.company.Company'>\r\nipdb> hasattr(collection, 'objects')\r\nFalse\r\nipdb> hasattr(collection, 'objects')\r\nTrue\r\n```\r\n\r\nCan this be caused by some lazy loading or some configuration issue?\r\n\r\nThank you!\n", "before_files": [{"content": "from bson import DBRef, SON\nimport six\n\nfrom mongoengine.base import (BaseDict, BaseList, EmbeddedDocumentList,\n                              TopLevelDocumentMetaclass, get_document)\nfrom mongoengine.base.datastructures import LazyReference\nfrom mongoengine.connection import get_db\nfrom mongoengine.document import Document, EmbeddedDocument\nfrom mongoengine.fields import DictField, ListField, MapField, ReferenceField\nfrom mongoengine.queryset import QuerySet\n\n\nclass DeReference(object):\n    def __call__(self, items, max_depth=1, instance=None, name=None):\n        \"\"\"\n        Cheaply dereferences the items to a set depth.\n        Also handles the conversion of complex data types.\n\n        :param items: The iterable (dict, list, queryset) to be dereferenced.\n        :param max_depth: The maximum depth to recurse to\n        :param instance: The owning instance used for tracking changes by\n            :class:`~mongoengine.base.ComplexBaseField`\n        :param name: The name of the field, used for tracking changes by\n            :class:`~mongoengine.base.ComplexBaseField`\n        :param get: A boolean determining if being called by __get__\n        \"\"\"\n        if items is None or isinstance(items, six.string_types):\n            return items\n\n        # cheapest way to convert a queryset to a list\n        # list(queryset) uses a count() query to determine length\n        if isinstance(items, QuerySet):\n            items = [i for i in items]\n\n        self.max_depth = max_depth\n        doc_type = None\n\n        if instance and isinstance(instance, (Document, EmbeddedDocument,\n                                              TopLevelDocumentMetaclass)):\n            doc_type = instance._fields.get(name)\n            while hasattr(doc_type, 'field'):\n                doc_type = doc_type.field\n\n            if isinstance(doc_type, ReferenceField):\n                field = doc_type\n                doc_type = doc_type.document_type\n                is_list = not hasattr(items, 'items')\n\n                if is_list and all([i.__class__ == doc_type for i in items]):\n                    return items\n                elif not is_list and all(\n                        [i.__class__ == doc_type for i in items.values()]):\n                    return items\n                elif not field.dbref:\n                    if not hasattr(items, 'items'):\n\n                        def _get_items(items):\n                            new_items = []\n                            for v in items:\n                                if isinstance(v, list):\n                                    new_items.append(_get_items(v))\n                                elif not isinstance(v, (DBRef, Document)):\n                                    new_items.append(field.to_python(v))\n                                else:\n                                    new_items.append(v)\n                            return new_items\n\n                        items = _get_items(items)\n                    else:\n                        items = {\n                            k: (v if isinstance(v, (DBRef, Document))\n                                else field.to_python(v))\n                            for k, v in items.iteritems()\n                        }\n\n        self.reference_map = self._find_references(items)\n        self.object_map = self._fetch_objects(doc_type=doc_type)\n        return self._attach_objects(items, 0, instance, name)\n\n    def _find_references(self, items, depth=0):\n        \"\"\"\n        Recursively finds all db references to be dereferenced\n\n        :param items: The iterable (dict, list, queryset)\n        :param depth: The current depth of recursion\n        \"\"\"\n        reference_map = {}\n        if not items or depth >= self.max_depth:\n            return reference_map\n\n        # Determine the iterator to use\n        if isinstance(items, dict):\n            iterator = items.values()\n        else:\n            iterator = items\n\n        # Recursively find dbreferences\n        depth += 1\n        for item in iterator:\n            if isinstance(item, (Document, EmbeddedDocument)):\n                for field_name, field in item._fields.iteritems():\n                    v = item._data.get(field_name, None)\n                    if isinstance(v, LazyReference):\n                        # LazyReference inherits DBRef but should not be dereferenced here !\n                        continue\n                    elif isinstance(v, DBRef):\n                        reference_map.setdefault(field.document_type, set()).add(v.id)\n                    elif isinstance(v, (dict, SON)) and '_ref' in v:\n                        reference_map.setdefault(get_document(v['_cls']), set()).add(v['_ref'].id)\n                    elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth:\n                        field_cls = getattr(getattr(field, 'field', None), 'document_type', None)\n                        references = self._find_references(v, depth)\n                        for key, refs in references.iteritems():\n                            if isinstance(field_cls, (Document, TopLevelDocumentMetaclass)):\n                                key = field_cls\n                            reference_map.setdefault(key, set()).update(refs)\n            elif isinstance(item, LazyReference):\n                # LazyReference inherits DBRef but should not be dereferenced here !\n                continue\n            elif isinstance(item, DBRef):\n                reference_map.setdefault(item.collection, set()).add(item.id)\n            elif isinstance(item, (dict, SON)) and '_ref' in item:\n                reference_map.setdefault(get_document(item['_cls']), set()).add(item['_ref'].id)\n            elif isinstance(item, (dict, list, tuple)) and depth - 1 <= self.max_depth:\n                references = self._find_references(item, depth - 1)\n                for key, refs in references.iteritems():\n                    reference_map.setdefault(key, set()).update(refs)\n\n        return reference_map\n\n    def _fetch_objects(self, doc_type=None):\n        \"\"\"Fetch all references and convert to their document objects\n        \"\"\"\n        object_map = {}\n        for collection, dbrefs in self.reference_map.iteritems():\n            if hasattr(collection, 'objects'):  # We have a document class for the refs\n                col_name = collection._get_collection_name()\n                refs = [dbref for dbref in dbrefs\n                        if (col_name, dbref) not in object_map]\n                references = collection.objects.in_bulk(refs)\n                for key, doc in references.iteritems():\n                    object_map[(col_name, key)] = doc\n            else:  # Generic reference: use the refs data to convert to document\n                if isinstance(doc_type, (ListField, DictField, MapField,)):\n                    continue\n\n                refs = [dbref for dbref in dbrefs\n                        if (collection, dbref) not in object_map]\n\n                if doc_type:\n                    references = doc_type._get_db()[collection].find({'_id': {'$in': refs}})\n                    for ref in references:\n                        doc = doc_type._from_son(ref)\n                        object_map[(collection, doc.id)] = doc\n                else:\n                    references = get_db()[collection].find({'_id': {'$in': refs}})\n                    for ref in references:\n                        if '_cls' in ref:\n                            doc = get_document(ref['_cls'])._from_son(ref)\n                        elif doc_type is None:\n                            doc = get_document(\n                                ''.join(x.capitalize()\n                                        for x in collection.split('_')))._from_son(ref)\n                        else:\n                            doc = doc_type._from_son(ref)\n                        object_map[(collection, doc.id)] = doc\n        return object_map\n\n    def _attach_objects(self, items, depth=0, instance=None, name=None):\n        \"\"\"\n        Recursively finds all db references to be dereferenced\n\n        :param items: The iterable (dict, list, queryset)\n        :param depth: The current depth of recursion\n        :param instance: The owning instance used for tracking changes by\n            :class:`~mongoengine.base.ComplexBaseField`\n        :param name: The name of the field, used for tracking changes by\n            :class:`~mongoengine.base.ComplexBaseField`\n        \"\"\"\n        if not items:\n            if isinstance(items, (BaseDict, BaseList)):\n                return items\n\n            if instance:\n                if isinstance(items, dict):\n                    return BaseDict(items, instance, name)\n                else:\n                    return BaseList(items, instance, name)\n\n        if isinstance(items, (dict, SON)):\n            if '_ref' in items:\n                return self.object_map.get(\n                    (items['_ref'].collection, items['_ref'].id), items)\n            elif '_cls' in items:\n                doc = get_document(items['_cls'])._from_son(items)\n                _cls = doc._data.pop('_cls', None)\n                del items['_cls']\n                doc._data = self._attach_objects(doc._data, depth, doc, None)\n                if _cls is not None:\n                    doc._data['_cls'] = _cls\n                return doc\n\n        if not hasattr(items, 'items'):\n            is_list = True\n            list_type = BaseList\n            if isinstance(items, EmbeddedDocumentList):\n                list_type = EmbeddedDocumentList\n            as_tuple = isinstance(items, tuple)\n            iterator = enumerate(items)\n            data = []\n        else:\n            is_list = False\n            iterator = items.iteritems()\n            data = {}\n\n        depth += 1\n        for k, v in iterator:\n            if is_list:\n                data.append(v)\n            else:\n                data[k] = v\n\n            if k in self.object_map and not is_list:\n                data[k] = self.object_map[k]\n            elif isinstance(v, (Document, EmbeddedDocument)):\n                for field_name in v._fields:\n                    v = data[k]._data.get(field_name, None)\n                    if isinstance(v, DBRef):\n                        data[k]._data[field_name] = self.object_map.get(\n                            (v.collection, v.id), v)\n                    elif isinstance(v, (dict, SON)) and '_ref' in v:\n                        data[k]._data[field_name] = self.object_map.get(\n                            (v['_ref'].collection, v['_ref'].id), v)\n                    elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth:\n                        item_name = six.text_type('{0}.{1}.{2}').format(name, k, field_name)\n                        data[k]._data[field_name] = self._attach_objects(v, depth, instance=instance, name=item_name)\n            elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth:\n                item_name = '%s.%s' % (name, k) if name else name\n                data[k] = self._attach_objects(v, depth - 1, instance=instance, name=item_name)\n            elif isinstance(v, DBRef) and hasattr(v, 'id'):\n                data[k] = self.object_map.get((v.collection, v.id), v)\n\n        if instance and name:\n            if is_list:\n                return tuple(data) if as_tuple else list_type(data, instance, name)\n            return BaseDict(data, instance, name)\n        depth += 1\n        return data\n", "path": "mongoengine/dereference.py"}], "after_files": [{"content": "from bson import DBRef, SON\nimport six\n\nfrom mongoengine.base import (BaseDict, BaseList, EmbeddedDocumentList,\n                              TopLevelDocumentMetaclass, get_document)\nfrom mongoengine.base.datastructures import LazyReference\nfrom mongoengine.connection import get_db\nfrom mongoengine.document import Document, EmbeddedDocument\nfrom mongoengine.fields import DictField, ListField, MapField, ReferenceField\nfrom mongoengine.queryset import QuerySet\n\n\nclass DeReference(object):\n    def __call__(self, items, max_depth=1, instance=None, name=None):\n        \"\"\"\n        Cheaply dereferences the items to a set depth.\n        Also handles the conversion of complex data types.\n\n        :param items: The iterable (dict, list, queryset) to be dereferenced.\n        :param max_depth: The maximum depth to recurse to\n        :param instance: The owning instance used for tracking changes by\n            :class:`~mongoengine.base.ComplexBaseField`\n        :param name: The name of the field, used for tracking changes by\n            :class:`~mongoengine.base.ComplexBaseField`\n        :param get: A boolean determining if being called by __get__\n        \"\"\"\n        if items is None or isinstance(items, six.string_types):\n            return items\n\n        # cheapest way to convert a queryset to a list\n        # list(queryset) uses a count() query to determine length\n        if isinstance(items, QuerySet):\n            items = [i for i in items]\n\n        self.max_depth = max_depth\n        doc_type = None\n\n        if instance and isinstance(instance, (Document, EmbeddedDocument,\n                                              TopLevelDocumentMetaclass)):\n            doc_type = instance._fields.get(name)\n            while hasattr(doc_type, 'field'):\n                doc_type = doc_type.field\n\n            if isinstance(doc_type, ReferenceField):\n                field = doc_type\n                doc_type = doc_type.document_type\n                is_list = not hasattr(items, 'items')\n\n                if is_list and all([i.__class__ == doc_type for i in items]):\n                    return items\n                elif not is_list and all(\n                        [i.__class__ == doc_type for i in items.values()]):\n                    return items\n                elif not field.dbref:\n                    if not hasattr(items, 'items'):\n\n                        def _get_items(items):\n                            new_items = []\n                            for v in items:\n                                if isinstance(v, list):\n                                    new_items.append(_get_items(v))\n                                elif not isinstance(v, (DBRef, Document)):\n                                    new_items.append(field.to_python(v))\n                                else:\n                                    new_items.append(v)\n                            return new_items\n\n                        items = _get_items(items)\n                    else:\n                        items = {\n                            k: (v if isinstance(v, (DBRef, Document))\n                                else field.to_python(v))\n                            for k, v in items.iteritems()\n                        }\n\n        self.reference_map = self._find_references(items)\n        self.object_map = self._fetch_objects(doc_type=doc_type)\n        return self._attach_objects(items, 0, instance, name)\n\n    def _find_references(self, items, depth=0):\n        \"\"\"\n        Recursively finds all db references to be dereferenced\n\n        :param items: The iterable (dict, list, queryset)\n        :param depth: The current depth of recursion\n        \"\"\"\n        reference_map = {}\n        if not items or depth >= self.max_depth:\n            return reference_map\n\n        # Determine the iterator to use\n        if isinstance(items, dict):\n            iterator = items.values()\n        else:\n            iterator = items\n\n        # Recursively find dbreferences\n        depth += 1\n        for item in iterator:\n            if isinstance(item, (Document, EmbeddedDocument)):\n                for field_name, field in item._fields.iteritems():\n                    v = item._data.get(field_name, None)\n                    if isinstance(v, LazyReference):\n                        # LazyReference inherits DBRef but should not be dereferenced here !\n                        continue\n                    elif isinstance(v, DBRef):\n                        reference_map.setdefault(field.document_type, set()).add(v.id)\n                    elif isinstance(v, (dict, SON)) and '_ref' in v:\n                        reference_map.setdefault(get_document(v['_cls']), set()).add(v['_ref'].id)\n                    elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth:\n                        field_cls = getattr(getattr(field, 'field', None), 'document_type', None)\n                        references = self._find_references(v, depth)\n                        for key, refs in references.iteritems():\n                            if isinstance(field_cls, (Document, TopLevelDocumentMetaclass)):\n                                key = field_cls\n                            reference_map.setdefault(key, set()).update(refs)\n            elif isinstance(item, LazyReference):\n                # LazyReference inherits DBRef but should not be dereferenced here !\n                continue\n            elif isinstance(item, DBRef):\n                reference_map.setdefault(item.collection, set()).add(item.id)\n            elif isinstance(item, (dict, SON)) and '_ref' in item:\n                reference_map.setdefault(get_document(item['_cls']), set()).add(item['_ref'].id)\n            elif isinstance(item, (dict, list, tuple)) and depth - 1 <= self.max_depth:\n                references = self._find_references(item, depth - 1)\n                for key, refs in references.iteritems():\n                    reference_map.setdefault(key, set()).update(refs)\n\n        return reference_map\n\n    def _fetch_objects(self, doc_type=None):\n        \"\"\"Fetch all references and convert to their document objects\n        \"\"\"\n        object_map = {}\n        for collection, dbrefs in self.reference_map.iteritems():\n\n            # we use getattr instead of hasattr because as hasattr swallows any exception under python2\n            # so it could hide nasty things without raising exceptions (cfr bug #1688))\n            ref_document_cls_exists = (getattr(collection, 'objects', None) is not None)\n\n            if ref_document_cls_exists:\n                col_name = collection._get_collection_name()\n                refs = [dbref for dbref in dbrefs\n                        if (col_name, dbref) not in object_map]\n                references = collection.objects.in_bulk(refs)\n                for key, doc in references.iteritems():\n                    object_map[(col_name, key)] = doc\n            else:  # Generic reference: use the refs data to convert to document\n                if isinstance(doc_type, (ListField, DictField, MapField,)):\n                    continue\n\n                refs = [dbref for dbref in dbrefs\n                        if (collection, dbref) not in object_map]\n\n                if doc_type:\n                    references = doc_type._get_db()[collection].find({'_id': {'$in': refs}})\n                    for ref in references:\n                        doc = doc_type._from_son(ref)\n                        object_map[(collection, doc.id)] = doc\n                else:\n                    references = get_db()[collection].find({'_id': {'$in': refs}})\n                    for ref in references:\n                        if '_cls' in ref:\n                            doc = get_document(ref['_cls'])._from_son(ref)\n                        elif doc_type is None:\n                            doc = get_document(\n                                ''.join(x.capitalize()\n                                        for x in collection.split('_')))._from_son(ref)\n                        else:\n                            doc = doc_type._from_son(ref)\n                        object_map[(collection, doc.id)] = doc\n        return object_map\n\n    def _attach_objects(self, items, depth=0, instance=None, name=None):\n        \"\"\"\n        Recursively finds all db references to be dereferenced\n\n        :param items: The iterable (dict, list, queryset)\n        :param depth: The current depth of recursion\n        :param instance: The owning instance used for tracking changes by\n            :class:`~mongoengine.base.ComplexBaseField`\n        :param name: The name of the field, used for tracking changes by\n            :class:`~mongoengine.base.ComplexBaseField`\n        \"\"\"\n        if not items:\n            if isinstance(items, (BaseDict, BaseList)):\n                return items\n\n            if instance:\n                if isinstance(items, dict):\n                    return BaseDict(items, instance, name)\n                else:\n                    return BaseList(items, instance, name)\n\n        if isinstance(items, (dict, SON)):\n            if '_ref' in items:\n                return self.object_map.get(\n                    (items['_ref'].collection, items['_ref'].id), items)\n            elif '_cls' in items:\n                doc = get_document(items['_cls'])._from_son(items)\n                _cls = doc._data.pop('_cls', None)\n                del items['_cls']\n                doc._data = self._attach_objects(doc._data, depth, doc, None)\n                if _cls is not None:\n                    doc._data['_cls'] = _cls\n                return doc\n\n        if not hasattr(items, 'items'):\n            is_list = True\n            list_type = BaseList\n            if isinstance(items, EmbeddedDocumentList):\n                list_type = EmbeddedDocumentList\n            as_tuple = isinstance(items, tuple)\n            iterator = enumerate(items)\n            data = []\n        else:\n            is_list = False\n            iterator = items.iteritems()\n            data = {}\n\n        depth += 1\n        for k, v in iterator:\n            if is_list:\n                data.append(v)\n            else:\n                data[k] = v\n\n            if k in self.object_map and not is_list:\n                data[k] = self.object_map[k]\n            elif isinstance(v, (Document, EmbeddedDocument)):\n                for field_name in v._fields:\n                    v = data[k]._data.get(field_name, None)\n                    if isinstance(v, DBRef):\n                        data[k]._data[field_name] = self.object_map.get(\n                            (v.collection, v.id), v)\n                    elif isinstance(v, (dict, SON)) and '_ref' in v:\n                        data[k]._data[field_name] = self.object_map.get(\n                            (v['_ref'].collection, v['_ref'].id), v)\n                    elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth:\n                        item_name = six.text_type('{0}.{1}.{2}').format(name, k, field_name)\n                        data[k]._data[field_name] = self._attach_objects(v, depth, instance=instance, name=item_name)\n            elif isinstance(v, (dict, list, tuple)) and depth <= self.max_depth:\n                item_name = '%s.%s' % (name, k) if name else name\n                data[k] = self._attach_objects(v, depth - 1, instance=instance, name=item_name)\n            elif isinstance(v, DBRef) and hasattr(v, 'id'):\n                data[k] = self.object_map.get((v.collection, v.id), v)\n\n        if instance and name:\n            if is_list:\n                return tuple(data) if as_tuple else list_type(data, instance, name)\n            return BaseDict(data, instance, name)\n        depth += 1\n        return data\n", "path": "mongoengine/dereference.py"}]}