<NME> db.py
<BEF> import itertools
import os
import ssl
import threading

import six
from six.moves import zip as izip
from Crypto import Random

import transaction
import ZODB
import ZODB.Connection

from zerodbext.catalog.query import optimize
from zerodb.collective.indexing.indexer import PortalCatalogProcessor
from zerodb.collective.indexing.interfaces import IIndexQueueProcessor
from zerodb.collective.indexing import subscribers
from zope import component

from zerodb import models
from zerodb.catalog.query import And, Eq
from zerodb.models.exceptions import ModelException
from zerodb.storage import client_storage
from zerodb.util.thread_watcher import ThreadWatcher
from zerodb.util.iter import DBList, DBListPrefetch, Sliceable

from zerodb.transform.encrypt_aes import AES256Encrypter, AES256EncrypterV0
from zerodb.transform import init_crypto
from zerodb.crypto import kdf


class AutoReindexQueueProcessor(PortalCatalogProcessor):
    def __init__(self, db, enabled=True):
        self.db = db
        self.enabled = enabled

    # execute reindex in before_commit hook when commit
    def reindex(self, obj, attributes=None):
        if self.enabled:
            self.db.reindex(obj, attributes)


class DbModel(object):
    """
    Class where model is combined with db.
    All functionality will actually reside here.
    Will contain indexes as well.
        if commit:
            transaction.commit()

    def __getitem__(self, oids):
        """
        DbModels (which we query) are accessed by using db as a dictionary

        :param int oids: object's uid or list of them
        :return: Persistent object(s)
        """
        if isinstance(oids, (int, long)):
            return self._objects[oids]
        elif isinstance(oids, (tuple, list, set)):
            return [self._objects[oid] for oid in oids]
        else:
            raise ModelException("Integer or list of integers is expected")

            db._root[self._intid_name] = _objects

        if self._catalog_name not in db._root:
            _catalog = model.create_catalog()
            db._root[self._catalog_name] = _catalog

        if commit:
            transaction.commit()

    @property
    def _catalog(self):
        return self._db._root[self._catalog_name]

    @property
    def _objects(self):
        return self._db._root[self._intid_name]

    def __getitem__(self, uids):
        """
        DbModels (which we query) are accessed by using db as a dictionary

        :param int uids: object's uid or list of them
        :return: Persistent object(s)
        """
        if isinstance(uids, six.integer_types):
            obj = self._objects[uids]
            if not hasattr(obj, "_p_uid"):
                obj._p_uid = uids
            return obj

        elif isinstance(uids, (tuple, list, set)):
            objects = [self._objects[uid] for uid in uids]
            self._db._connection.prefetch(objects)
            for o, uid in izip(objects, uids):
                if not hasattr(o, "_p_uid"):
                    o._p_uid = uid
            return objects

        else:
            raise ModelException("Integer or list of integers is expected")

    def all_uids(self):
        for i in self._objects.tree:
            yield i

    def all(self):
        for i in self.all_uids():
            obj = self._objects[i]
            if not hasattr(obj, "_p_uid"):
                obj._p_uid = i
            yield obj

    def add(self, obj):
        """
        Add newly created a Model object to the database
        Stores *and* indexes it

        :param zerodb.models.Model obj: Object to add to the database
        :return: Added object's uid
        :rtype: int
        """
        assert obj.__class__ == self._model
        uid = self._objects.add(obj)
        self._catalog.index_doc(uid, obj)
        obj._p_uid = uid
        return uid

    def reindex_one(self, obj, attributes=None):
        """
        Reindex one object which is already in the database

        :param obj: Object to add to the database or its uid
        :type obj: zerodb.models.Model, int
        :param attributes: Attributes of obj to be reindex
        :type attributes: tuple, list
        """

        if isinstance(obj, six.integer_types):
            uid = obj
            obj = self._objects[uid]
        elif isinstance(obj, self._model):
            if hasattr(obj, "_p_uid"):
                uid = obj._p_uid
            else:
                raise ModelException("Object %s is not indexed" % obj)
        else:
            raise TypeError("Wrong type of argument passed: obj must be integer or model instance")

        if attributes is None:
            self._catalog.reindex_doc(uid, obj)
        elif isinstance(attributes, (tuple, list)):
            for attr in attributes:
                if attr in self._catalog:
                    self._catalog[attr].reindex_doc(uid, obj)
        else:
            raise TypeError("Wrong type of argument passed: attributes must be tuple or list")

    def reindex(self, obj):
        """
        Reindex one or multiple objects in the database

        :param obj: Object to add to the database or its uid, or list of objects or uids
        :type obj: zerodb.models.Model, int, list
        """
        if isinstance(obj, six.integer_types) or isinstance(obj, self._model):
            self.reindex_one(obj)
        elif isinstance(obj, (list, tuple, set, Sliceable)):
            for o in obj:
                self.reindex_one(o)
        else:
            raise TypeError("ZeroDB object, its uid or list of these should be passed")

    def remove(self, obj):
        """
        Remove existing object from the database + unindex it

        :param zerodb.models.Model obj: Object to add to the database
        """
        if isinstance(obj, six.integer_types):
            uid = obj
        elif hasattr(obj, "__iter__"):
            ctr = 0
            for i in list(obj):
                self.remove(i)
                ctr += 1
            return ctr
        else:
            assert obj.__class__ == self._model
            uid = obj._p_uid
        self._catalog.unindex_doc(uid)
        del self._objects[uid]
        return 1

    def query(self, queryobj=None, skip=None, limit=None, prefetch=True, **kw):
        """
        Smart proxy to catalog's query.
        One can add <field=...> keyword arguments to make queries where fields
        are equal to specified values

        :param zerodb.catalog.query.Query queryobj: Query which all sorts of
            logical, range queries etc
        :param int skip: Offset to start the result iteration from
        :param int limit: Limit number of results to this
        """
        # Catalog's query returns only integers
        # We must be smart here and return objects
        # But no, we must be even smarter and batch-preload objects
        # Most difficult part is preloading TreeSets for index when needed
        # (when we do complex queries which require composite index)
        # We also probably should do something like lazy query(...)[ofs:...]
        # if no limit, skip are used

        # Work needed on skip and limit because zope didn't well support them...
        skip = skip or 0
        if limit:
            kw["limit"] = skip + limit

        eq_args = []
        for k in list(kw.keys()):
            if k not in set(["sort_index", "sort_type", "reverse", "names", "limit"]):
                eq_args.append(Eq(k, kw.pop(k)))

        if queryobj:
            Q = optimize(optimize(queryobj) & And(*eq_args))
        else:
            Q = And(*eq_args)

        q = lambda: self._catalog.query(Q, **kw)

        if limit:
            _, q = q()
            # XXX islice -> [:]
            qids = list(itertools.islice(q, skip, skip + limit))
            objects = [self._objects[uid] for uid in qids]
            if objects and prefetch:
                self._db._connection.prefetch(objects)
            for obj, uid in izip(objects, qids):
                obj._p_uid = uid
            return objects

        else:
            db_list = DBListPrefetch if prefetch else DBList
            return db_list(q, self)

    def __len__(self):
        return len(self._objects)


class SubConnection(ZODB.Connection.Connection):

    @property
    def root(self):
        return ZODB.Connection.RootConvenience(self.get(self._db._root_oid))

    def setstate(self, obj):
        if hasattr(obj, "_p_uid"):
            uid = obj._p_uid
        else:
            uid = None
        super(SubConnection, self).setstate(obj)
        if uid is not None:
            obj._p_uid = uid


class SubDB(ZODB.DB):
    klass = SubConnection

    def __init__(self, storage, *a, **kw):
        super(SubDB, self).__init__(storage, *a, **kw)
        self._root_oid = storage.get_root_id()


class DB(object):
    """
    Database for this user. Everything is used through this class
    """

    encrypter = [AES256Encrypter, AES256EncrypterV0]
    appname = 'zerodb.com'
    compressor = None

    def __init__(self, sock, key=None, username=None, password=None,
                 cert_file=None, key_file=None, server_cert=None,
                 security=None,
                 debug=False, pool_timeout=3600, pool_size=7,
                 autoreindex=True, wait_timeout=30,
                 **kw):
        """
        :param str sock: UNIX (str) or TCP ((str, int)) socket
        :type sock: str or tuple
        :param str username: Username
        :param str password: Password
        :param bytes key: Encryption key

        :param cert_file: Client certificate for authentication (pem)
        :param key_file: Private key for that certificate (pem)
        :param server_cert: Server certitificate if not registered with CA

        :param function security: Key derivation function from
                                  zerodb.crypto.kdf

        :param bool debug: Whether to log debug messages
        """

        if (cert_file or key_file) and not (cert_file and key_file):
            raise TypeError("If you specify a cert file or a key file,"
                            " you must specify both.")

        ssl_context = make_ssl(cert_file, key_file, server_cert)

        if security is None:
            security = kdf.guess(
                    username, password, key_file, cert_file, self.appname, key)

        password, key = security(
                username, password, key_file, cert_file, self.appname, key)

        if password:
            credentials = dict(name=username, password=password)
        else:
            credentials = None

        if isinstance(sock, six.string_types):
            sock = str(sock)
        elif type(sock) in (list, tuple):
            assert len(sock) == 2
            sock = str(sock[0]), int(sock[1])

        self._autoreindex = autoreindex
        self._reindex_queue_processor = AutoReindexQueueProcessor(
            self, enabled=autoreindex)
        component.provideUtility(
            self._reindex_queue_processor,
            IIndexQueueProcessor,
            'zerodb-indexer')

        self._init_default_crypto(key=key)

        # Store all the arguments necessary for login in this instance
        self.__storage_kwargs = {
                "sock": sock,
                "ssl": ssl_context,
                "cache_size": 2 ** 30,
                "debug": debug,
                "wait_timeout": wait_timeout,
                "credentials": credentials,
            }

        self.__db_kwargs = {
                "pool_size": pool_size,
                "pool_timeout": pool_timeout,
                "cache_size": 1000000,
                "cache_size_bytes": 100 * 2 ** 20}
        self.__db_kwargs.update(kw)

        # For multi-threading
        self.__pid = os.getpid()

        self._init_db()
        self._models = {}

    @classmethod
    def _init_default_crypto(self, **kw):
        encrypters = self.encrypter
        if not isinstance(encrypters, (list, tuple)):
            encrypters = [self.encrypter]
        elif not encrypters:
            encrypters = []

        if encrypters:
            encrypters[0].register_class(default=True)
            for e in encrypters[1:]:
                e.register_class(default=False)
        if self.compressor:
            self.compressor.register(default=True)

        init_crypto(**kw)

    def _init_db(self):
        """We need this to be executed each time we are in a new process"""
        if self._autoreindex:
            subscribers.init()

        Random.atfork()

        self.__conn_refs = {}
        self.__thread_local = threading.local()
        self.__thread_watcher = ThreadWatcher()
        self._storage = client_storage(**self.__storage_kwargs)
        self._db = SubDB(self._storage, **self.__db_kwargs)
        self._conn_open()

    def _conn_open(self):
        """Opens db connection and registers a destuction callback"""

        self.__thread_local.conn = conn = self._db.open()

        def destructor(conn_id):
            conn = self._db.pool.all.data.get(conn_id, None)
            if conn:
                # Hack to make it not closing TM which is already removed
                conn.opened = 0
                conn.close()

        self.__thread_watcher.watch(destructor, id(conn))

    @property
    def _root(self):
        """Access database root for this user"""

        if os.getpid() != self.__pid:
            # If a new process spins up, we need to re-initialize everything
            self.__pid = os.getpid()
            self._init_db()
        else:
            # Open connections from the pool when new threads spin up
            # Should be closed when old thread-locals get garbage collected
            if not hasattr(self.__thread_local, "conn") or\
                    self.__thread_local.conn.opened is None:
                self._conn_open()

        return self.__thread_local.conn.root()

    @property
    def _connection(self):
        return self.__thread_local.conn

    def disconnect(self):
        if hasattr(self.__thread_local, "conn"):
            self.__thread_local.conn.close()

    def __getitem__(self, model):
        """
        DbModels (which we query) are accessed by using db as a dictionary

        :param model: Subclass of zerodb.models.Model to return or create db entry for
        :rtype: zerodb.db.DbModel
        """
        # TODO implement list of keys, writing to arbitrary (new) dbmodel (which is not defined)
        if not issubclass(model, models.Model):
            raise ModelException("Class <%s> is not a Model" % model.__name__)
        if model not in self._models:
            self._models[model] = DbModel(self, model)
        return self._models[model]

    def add(self, obj):
        """
        Add newly created a Model object to the database
        Stores *and* indexes it

        :param zerodb.models.Model obj: Object to add to the database
        :return: Added object's uid
        :rtype: int
        """
        if isinstance(obj, (list, set, tuple)):
            return [self[o.__class__].add(o) for o in obj]
        else:
            return self[obj.__class__].add(obj)

    def remove(self, obj):
        """
        Remove existing object from the database + unindex it

        :param zerodb.models.Model obj: Object to add to the database
        """
        if isinstance(obj, models.Model):
            self[obj.__class__].remove(obj)
            return 1
        elif hasattr(obj, "__iter__"):
            ctr = 0
            for o in obj:
                ctr += 1
                self[o.__class__].remove(o)
            return ctr
        else:
            raise ModelException("Class <%s> is not a Model or iterable" % obj.__class__.__name__)

    def reindex(self, obj, attributes=None):
        """
        Reindex one or multiple objects in the database

        :param obj: Object to add to the database or its uid, or list of objects or uids
        :type obj: zerodb.models.Model, list
        :param attributes: Attributes of obj to be reindex
        :type attributes: tuple, list
        """
        if isinstance(obj, models.Model):
            self[obj.__class__].reindex_one(obj, attributes)
        elif isinstance(obj, (list, tuple, set, Sliceable)):
            for o in obj:
                assert isinstance(o, models.Model)
                self[o.__class__].reindex_one(o, attributes)
        else:
            raise TypeError("ZeroDB object or list of these should be passed")

    def pack(self):
        """
        Remove old versions of objects
        """
        self._db.pack()

    def enableAutoReindex(self, enabled=True):
        """
        Enable or disable auto reindex
        """
        if enabled:
            subscribers.init()
        self._reindex_queue_processor.enabled = enabled


def make_ssl(cert_file=None, key_file=None, server_cert=None):
    ssl_context = ssl.create_default_context(
        ssl.Purpose.CLIENT_AUTH, cafile=server_cert)
    here = os.path.dirname(__file__)
    ssl_context.load_cert_chain(
        cert_file or os.path.join(here, 'permissions/nobody.pem'),
        key_file or os.path.join(here, 'permissions/nobody-key.pem'),
        )
    ssl_context.verify_mode = ssl.CERT_REQUIRED
    if server_cert is None:
        ssl_context.check_hostname = True
        ssl_context.load_default_certs()
    else:
        ssl_context.check_hostname = False
    return ssl_context

<MSG> Get by ID api

<DFF> @@ -48,17 +48,19 @@ class DbModel(object):
         if commit:
             transaction.commit()
 
-    def __getitem__(self, oids):
+    def __getitem__(self, uids):
         """
         DbModels (which we query) are accessed by using db as a dictionary
 
-        :param int oids: object's uid or list of them
+        :param int uids: object's uid or list of them
         :return: Persistent object(s)
         """
-        if isinstance(oids, (int, long)):
-            return self._objects[oids]
-        elif isinstance(oids, (tuple, list, set)):
-            return [self._objects[oid] for oid in oids]
+        if isinstance(uids, (int, long)):
+            return self._objects[uids]
+        elif isinstance(uids, (tuple, list, set)):
+            objects = [self._objects[uid] for uid in uids]
+            self._db._storage.loadBulk([o._p_oid for o in objects])
+            return objects
         else:
             raise ModelException("Integer or list of integers is expected")
 

{'commit': 'diff --git a/zerodb/api.py b/zerodb/api.py\nindex c35323b..bd7acbb 100644\n--- a/zerodb/api.py\n+++ b/zerodb/api.py\n@@ -108,6 +108,23 @@ def insert(table_name):\n         return jsonify(ok=0)\n \n \n+@app.route("/<table_name>/_get", methods=["GET", "POST"])\n+def get(table_name):\n+    db = dbs[session["username"]]\n+    model = getattr(models, table_name)\n+\n+    if request.method == "GET":\n+        req = request.args\n+    elif request.method == "POST":\n+        req = request.form\n+    else:\n+        return jsonify(ok=0)\n+\n+    data = json.loads(req["_id"])\n+\n+    return jsonpickle.encode(db[model][data], unpicklable=False)\n+\n+\n @app.route("/_disconnect", methods=["GET", "POST"])\n def disconnect():\n     if "username" in session:\ndiff --git a/zerodb/db.py b/zerodb/db.py\nindex 25c6a9d..19c8d14 100644\n--- a/zerodb/db.py\n+++ b/zerodb/db.py\n@@ -48,17 +48,19 @@ class DbModel(object):\n         if commit:\n             transaction.commit()\n \n-    def __getitem__(self, oids):\n+    def __getitem__(self, uids):\n         """\n         DbModels (which we query) are accessed by using db as a dictionary\n \n-        :param int oids: object\'s uid or list of them\n+        :param int uids: object\'s uid or list of them\n         :return: Persistent object(s)\n         """\n-        if isinstance(oids, (int, long)):\n-            return self._objects[oids]\n-        elif isinstance(oids, (tuple, list, set)):\n-            return [self._objects[oid] for oid in oids]\n+        if isinstance(uids, (int, long)):\n+            return self._objects[uids]\n+        elif isinstance(uids, (tuple, list, set)):\n+            objects = [self._objects[uid] for uid in uids]\n+            self._db._storage.loadBulk([o._p_oid for o in objects])\n+            return objects\n         else:\n             raise ModelException("Integer or list of integers is expected")\n \n', 'commit_subject': 'Get by ID api', 'repo_name': 'zerodb/zerodb', 'committer': {'name': 'Michael Egorov', 'email': '5e74db1f2273065a15b5d0c69c12827c20f86755@gmail.com', 'time_sec': '1435434959', 'tz_offset': '-420', 'date': {'seconds': '1435434959'}}, 'commit_tree': '0b9e465d118a700785c6616128fc5650ac31e78e', 'commit_message': 'Get by ID api\n', 'language_name': 'Shell', 'license': 'agpl-3.0', 'hunks': '@@ -48,17 +48,19 @@ class DbModel(object):\n         if commit:\n             transaction.commit()\n \n-    def __getitem__(self, oids):\n+    def __getitem__(self, uids):\n         """\n         DbModels (which we query) are accessed by using db as a dictionary\n \n-        :param int oids: object\'s uid or list of them\n+        :param int uids: object\'s uid or list of them\n         :return: Persistent object(s)\n         """\n-        if isinstance(oids, (int, long)):\n-            return self._objects[oids]\n-        elif isinstance(oids, (tuple, list, set)):\n-            return [self._objects[oid] for oid in oids]\n+        if isinstance(uids, (int, long)):\n+            return self._objects[uids]\n+        elif isinstance(uids, (tuple, list, set)):\n+            objects = [self._objects[uid] for uid in uids]\n+            self._db._storage.loadBulk([o._p_oid for o in objects])\n+            return objects\n         else:\n             raise ModelException("Integer or list of integers is expected")\n \n', 'addition_count': 8, 'deletion_count': 6, 'src_file': 'a/zerodb/db.py', 'tgt_file': 'b/zerodb/db.py', 'file_extension': '.py', 'before_file': 'import itertools\nimport os\nimport ssl\nimport threading\n\nimport six\nfrom six.moves import zip as izip\nfrom Crypto import Random\n\nimport transaction\nimport ZODB\nimport ZODB.Connection\n\nfrom zerodbext.catalog.query import optimize\nfrom zerodb.collective.indexing.indexer import PortalCatalogProcessor\nfrom zerodb.collective.indexing.interfaces import IIndexQueueProcessor\nfrom zerodb.collective.indexing import subscribers\nfrom zope import component\n\nfrom zerodb import models\nfrom zerodb.catalog.query import And, Eq\nfrom zerodb.models.exceptions import ModelException\nfrom zerodb.storage import client_storage\nfrom zerodb.util.thread_watcher import ThreadWatcher\nfrom zerodb.util.iter import DBList, DBListPrefetch, Sliceable\n\nfrom zerodb.transform.encrypt_aes import AES256Encrypter, AES256EncrypterV0\nfrom zerodb.transform import init_crypto\nfrom zerodb.crypto import kdf\n\n\nclass AutoReindexQueueProcessor(PortalCatalogProcessor):\n    def __init__(self, db, enabled=True):\n        self.db = db\n        self.enabled = enabled\n\n    # execute reindex in before_commit hook when commit\n    def reindex(self, obj, attributes=None):\n        if self.enabled:\n            self.db.reindex(obj, attributes)\n\n\nclass DbModel(object):\n    """\n    Class where model is combined with db.\n    All functionality will actually reside here.\n    Will contain indexes as well.\n        if commit:\n            transaction.commit()\n\n    def __getitem__(self, oids):\n        """\n        DbModels (which we query) are accessed by using db as a dictionary\n\n        :param int oids: object\'s uid or list of them\n        :return: Persistent object(s)\n        """\n        if isinstance(oids, (int, long)):\n            return self._objects[oids]\n        elif isinstance(oids, (tuple, list, set)):\n            return [self._objects[oid] for oid in oids]\n        else:\n            raise ModelException("Integer or list of integers is expected")\n\n            db._root[self._intid_name] = _objects\n\n        if self._catalog_name not in db._root:\n            _catalog = model.create_catalog()\n            db._root[self._catalog_name] = _catalog\n\n        if commit:\n            transaction.commit()\n\n    @property\n    def _catalog(self):\n        return self._db._root[self._catalog_name]\n\n    @property\n    def _objects(self):\n        return self._db._root[self._intid_name]\n\n    def __getitem__(self, uids):\n        """\n        DbModels (which we query) are accessed by using db as a dictionary\n\n        :param int uids: object\'s uid or list of them\n        :return: Persistent object(s)\n        """\n        if isinstance(uids, six.integer_types):\n            obj = self._objects[uids]\n            if not hasattr(obj, "_p_uid"):\n                obj._p_uid = uids\n            return obj\n\n        elif isinstance(uids, (tuple, list, set)):\n            objects = [self._objects[uid] for uid in uids]\n            self._db._connection.prefetch(objects)\n            for o, uid in izip(objects, uids):\n                if not hasattr(o, "_p_uid"):\n                    o._p_uid = uid\n            return objects\n\n        else:\n            raise ModelException("Integer or list of integers is expected")\n\n    def all_uids(self):\n        for i in self._objects.tree:\n            yield i\n\n    def all(self):\n        for i in self.all_uids():\n            obj = self._objects[i]\n            if not hasattr(obj, "_p_uid"):\n                obj._p_uid = i\n            yield obj\n\n    def add(self, obj):\n        """\n        Add newly created a Model object to the database\n        Stores *and* indexes it\n\n        :param zerodb.models.Model obj: Object to add to the database\n        :return: Added object\'s uid\n        :rtype: int\n        """\n        assert obj.__class__ == self._model\n        uid = self._objects.add(obj)\n        self._catalog.index_doc(uid, obj)\n        obj._p_uid = uid\n        return uid\n\n    def reindex_one(self, obj, attributes=None):\n        """\n        Reindex one object which is already in the database\n\n        :param obj: Object to add to the database or its uid\n        :type obj: zerodb.models.Model, int\n        :param attributes: Attributes of obj to be reindex\n        :type attributes: tuple, list\n        """\n\n        if isinstance(obj, six.integer_types):\n            uid = obj\n            obj = self._objects[uid]\n        elif isinstance(obj, self._model):\n            if hasattr(obj, "_p_uid"):\n                uid = obj._p_uid\n            else:\n                raise ModelException("Object %s is not indexed" % obj)\n        else:\n            raise TypeError("Wrong type of argument passed: obj must be integer or model instance")\n\n        if attributes is None:\n            self._catalog.reindex_doc(uid, obj)\n        elif isinstance(attributes, (tuple, list)):\n            for attr in attributes:\n                if attr in self._catalog:\n                    self._catalog[attr].reindex_doc(uid, obj)\n        else:\n            raise TypeError("Wrong type of argument passed: attributes must be tuple or list")\n\n    def reindex(self, obj):\n        """\n        Reindex one or multiple objects in the database\n\n        :param obj: Object to add to the database or its uid, or list of objects or uids\n        :type obj: zerodb.models.Model, int, list\n        """\n        if isinstance(obj, six.integer_types) or isinstance(obj, self._model):\n            self.reindex_one(obj)\n        elif isinstance(obj, (list, tuple, set, Sliceable)):\n            for o in obj:\n                self.reindex_one(o)\n        else:\n            raise TypeError("ZeroDB object, its uid or list of these should be passed")\n\n    def remove(self, obj):\n        """\n        Remove existing object from the database + unindex it\n\n        :param zerodb.models.Model obj: Object to add to the database\n        """\n        if isinstance(obj, six.integer_types):\n            uid = obj\n        elif hasattr(obj, "__iter__"):\n            ctr = 0\n            for i in list(obj):\n                self.remove(i)\n                ctr += 1\n            return ctr\n        else:\n            assert obj.__class__ == self._model\n            uid = obj._p_uid\n        self._catalog.unindex_doc(uid)\n        del self._objects[uid]\n        return 1\n\n    def query(self, queryobj=None, skip=None, limit=None, prefetch=True, **kw):\n        """\n        Smart proxy to catalog\'s query.\n        One can add <field=...> keyword arguments to make queries where fields\n        are equal to specified values\n\n        :param zerodb.catalog.query.Query queryobj: Query which all sorts of\n            logical, range queries etc\n        :param int skip: Offset to start the result iteration from\n        :param int limit: Limit number of results to this\n        """\n        # Catalog\'s query returns only integers\n        # We must be smart here and return objects\n        # But no, we must be even smarter and batch-preload objects\n        # Most difficult part is preloading TreeSets for index when needed\n        # (when we do complex queries which require composite index)\n        # We also probably should do something like lazy query(...)[ofs:...]\n        # if no limit, skip are used\n\n        # Work needed on skip and limit because zope didn\'t well support them...\n        skip = skip or 0\n        if limit:\n            kw["limit"] = skip + limit\n\n        eq_args = []\n        for k in list(kw.keys()):\n            if k not in set(["sort_index", "sort_type", "reverse", "names", "limit"]):\n                eq_args.append(Eq(k, kw.pop(k)))\n\n        if queryobj:\n            Q = optimize(optimize(queryobj) & And(*eq_args))\n        else:\n            Q = And(*eq_args)\n\n        q = lambda: self._catalog.query(Q, **kw)\n\n        if limit:\n            _, q = q()\n            # XXX islice -> [:]\n            qids = list(itertools.islice(q, skip, skip + limit))\n            objects = [self._objects[uid] for uid in qids]\n            if objects and prefetch:\n                self._db._connection.prefetch(objects)\n            for obj, uid in izip(objects, qids):\n                obj._p_uid = uid\n            return objects\n\n        else:\n            db_list = DBListPrefetch if prefetch else DBList\n            return db_list(q, self)\n\n    def __len__(self):\n        return len(self._objects)\n\n\nclass SubConnection(ZODB.Connection.Connection):\n\n    @property\n    def root(self):\n        return ZODB.Connection.RootConvenience(self.get(self._db._root_oid))\n\n    def setstate(self, obj):\n        if hasattr(obj, "_p_uid"):\n            uid = obj._p_uid\n        else:\n            uid = None\n        super(SubConnection, self).setstate(obj)\n        if uid is not None:\n            obj._p_uid = uid\n\n\nclass SubDB(ZODB.DB):\n    klass = SubConnection\n\n    def __init__(self, storage, *a, **kw):\n        super(SubDB, self).__init__(storage, *a, **kw)\n        self._root_oid = storage.get_root_id()\n\n\nclass DB(object):\n    """\n    Database for this user. Everything is used through this class\n    """\n\n    encrypter = [AES256Encrypter, AES256EncrypterV0]\n    appname = \'zerodb.com\'\n    compressor = None\n\n    def __init__(self, sock, key=None, username=None, password=None,\n                 cert_file=None, key_file=None, server_cert=None,\n                 security=None,\n                 debug=False, pool_timeout=3600, pool_size=7,\n                 autoreindex=True, wait_timeout=30,\n                 **kw):\n        """\n        :param str sock: UNIX (str) or TCP ((str, int)) socket\n        :type sock: str or tuple\n        :param str username: Username\n        :param str password: Password\n        :param bytes key: Encryption key\n\n        :param cert_file: Client certificate for authentication (pem)\n        :param key_file: Private key for that certificate (pem)\n        :param server_cert: Server certitificate if not registered with CA\n\n        :param function security: Key derivation function from\n                                  zerodb.crypto.kdf\n\n        :param bool debug: Whether to log debug messages\n        """\n\n        if (cert_file or key_file) and not (cert_file and key_file):\n            raise TypeError("If you specify a cert file or a key file,"\n                            " you must specify both.")\n\n        ssl_context = make_ssl(cert_file, key_file, server_cert)\n\n        if security is None:\n            security = kdf.guess(\n                    username, password, key_file, cert_file, self.appname, key)\n\n        password, key = security(\n                username, password, key_file, cert_file, self.appname, key)\n\n        if password:\n            credentials = dict(name=username, password=password)\n        else:\n            credentials = None\n\n        if isinstance(sock, six.string_types):\n            sock = str(sock)\n        elif type(sock) in (list, tuple):\n            assert len(sock) == 2\n            sock = str(sock[0]), int(sock[1])\n\n        self._autoreindex = autoreindex\n        self._reindex_queue_processor = AutoReindexQueueProcessor(\n            self, enabled=autoreindex)\n        component.provideUtility(\n            self._reindex_queue_processor,\n            IIndexQueueProcessor,\n            \'zerodb-indexer\')\n\n        self._init_default_crypto(key=key)\n\n        # Store all the arguments necessary for login in this instance\n        self.__storage_kwargs = {\n                "sock": sock,\n                "ssl": ssl_context,\n                "cache_size": 2 ** 30,\n                "debug": debug,\n                "wait_timeout": wait_timeout,\n                "credentials": credentials,\n            }\n\n        self.__db_kwargs = {\n                "pool_size": pool_size,\n                "pool_timeout": pool_timeout,\n                "cache_size": 1000000,\n                "cache_size_bytes": 100 * 2 ** 20}\n        self.__db_kwargs.update(kw)\n\n        # For multi-threading\n        self.__pid = os.getpid()\n\n        self._init_db()\n        self._models = {}\n\n    @classmethod\n    def _init_default_crypto(self, **kw):\n        encrypters = self.encrypter\n        if not isinstance(encrypters, (list, tuple)):\n            encrypters = [self.encrypter]\n        elif not encrypters:\n            encrypters = []\n\n        if encrypters:\n            encrypters[0].register_class(default=True)\n            for e in encrypters[1:]:\n                e.register_class(default=False)\n        if self.compressor:\n            self.compressor.register(default=True)\n\n        init_crypto(**kw)\n\n    def _init_db(self):\n        """We need this to be executed each time we are in a new process"""\n        if self._autoreindex:\n            subscribers.init()\n\n        Random.atfork()\n\n        self.__conn_refs = {}\n        self.__thread_local = threading.local()\n        self.__thread_watcher = ThreadWatcher()\n        self._storage = client_storage(**self.__storage_kwargs)\n        self._db = SubDB(self._storage, **self.__db_kwargs)\n        self._conn_open()\n\n    def _conn_open(self):\n        """Opens db connection and registers a destuction callback"""\n\n        self.__thread_local.conn = conn = self._db.open()\n\n        def destructor(conn_id):\n            conn = self._db.pool.all.data.get(conn_id, None)\n            if conn:\n                # Hack to make it not closing TM which is already removed\n                conn.opened = 0\n                conn.close()\n\n        self.__thread_watcher.watch(destructor, id(conn))\n\n    @property\n    def _root(self):\n        """Access database root for this user"""\n\n        if os.getpid() != self.__pid:\n            # If a new process spins up, we need to re-initialize everything\n            self.__pid = os.getpid()\n            self._init_db()\n        else:\n            # Open connections from the pool when new threads spin up\n            # Should be closed when old thread-locals get garbage collected\n            if not hasattr(self.__thread_local, "conn") or\\\n                    self.__thread_local.conn.opened is None:\n                self._conn_open()\n\n        return self.__thread_local.conn.root()\n\n    @property\n    def _connection(self):\n        return self.__thread_local.conn\n\n    def disconnect(self):\n        if hasattr(self.__thread_local, "conn"):\n            self.__thread_local.conn.close()\n\n    def __getitem__(self, model):\n        """\n        DbModels (which we query) are accessed by using db as a dictionary\n\n        :param model: Subclass of zerodb.models.Model to return or create db entry for\n        :rtype: zerodb.db.DbModel\n        """\n        # TODO implement list of keys, writing to arbitrary (new) dbmodel (which is not defined)\n        if not issubclass(model, models.Model):\n            raise ModelException("Class <%s> is not a Model" % model.__name__)\n        if model not in self._models:\n            self._models[model] = DbModel(self, model)\n        return self._models[model]\n\n    def add(self, obj):\n        """\n        Add newly created a Model object to the database\n        Stores *and* indexes it\n\n        :param zerodb.models.Model obj: Object to add to the database\n        :return: Added object\'s uid\n        :rtype: int\n        """\n        if isinstance(obj, (list, set, tuple)):\n            return [self[o.__class__].add(o) for o in obj]\n        else:\n            return self[obj.__class__].add(obj)\n\n    def remove(self, obj):\n        """\n        Remove existing object from the database + unindex it\n\n        :param zerodb.models.Model obj: Object to add to the database\n        """\n        if isinstance(obj, models.Model):\n            self[obj.__class__].remove(obj)\n            return 1\n        elif hasattr(obj, "__iter__"):\n            ctr = 0\n            for o in obj:\n                ctr += 1\n                self[o.__class__].remove(o)\n            return ctr\n        else:\n            raise ModelException("Class <%s> is not a Model or iterable" % obj.__class__.__name__)\n\n    def reindex(self, obj, attributes=None):\n        """\n        Reindex one or multiple objects in the database\n\n        :param obj: Object to add to the database or its uid, or list of objects or uids\n        :type obj: zerodb.models.Model, list\n        :param attributes: Attributes of obj to be reindex\n        :type attributes: tuple, list\n        """\n        if isinstance(obj, models.Model):\n            self[obj.__class__].reindex_one(obj, attributes)\n        elif isinstance(obj, (list, tuple, set, Sliceable)):\n            for o in obj:\n                assert isinstance(o, models.Model)\n                self[o.__class__].reindex_one(o, attributes)\n        else:\n            raise TypeError("ZeroDB object or list of these should be passed")\n\n    def pack(self):\n        """\n        Remove old versions of objects\n        """\n        self._db.pack()\n\n    def enableAutoReindex(self, enabled=True):\n        """\n        Enable or disable auto reindex\n        """\n        if enabled:\n            subscribers.init()\n        self._reindex_queue_processor.enabled = enabled\n\n\ndef make_ssl(cert_file=None, key_file=None, server_cert=None):\n    ssl_context = ssl.create_default_context(\n        ssl.Purpose.CLIENT_AUTH, cafile=server_cert)\n    here = os.path.dirname(__file__)\n    ssl_context.load_cert_chain(\n        cert_file or os.path.join(here, \'permissions/nobody.pem\'),\n        key_file or os.path.join(here, \'permissions/nobody-key.pem\'),\n        )\n    ssl_context.verify_mode = ssl.CERT_REQUIRED\n    if server_cert is None:\n        ssl_context.check_hostname = True\n        ssl_context.load_default_certs()\n    else:\n        ssl_context.check_hostname = False\n    return ssl_context\n', 'message': 'Get by ID api\n', 'author_name': 'Michael Egorov', 'lang': 'py'}