files: import array
  import datetime
  import itertools
  import logging
  import md5
  import random
  import sys
  import threading
  import time
  
  import dbconstants
  import helper_functions
  
  from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
  from kazoo.client import KazooState
  from .dbconstants import APP_ENTITY_SCHEMA
  from .dbconstants import ID_KEY_LENGTH
  from .dbconstants import MAX_TX_DURATION
  from .cassandra_env import cassandra_interface
  from .cassandra_env.entity_id_allocator import EntityIDAllocator
  from .cassandra_env.entity_id_allocator import ScatteredAllocator
  from .cassandra_env.utils import deletions_for_entity
  from .cassandra_env.utils import mutations_for_entity
  from .utils import clean_app_id
  from .utils import encode_entity_table_key
  from .utils import encode_index_pb
  from .utils import encode_path_from_filter
  from .utils import get_composite_index_keys
  from .utils import get_entity_key
  from .utils import get_entity_kind
  from .utils import get_index_key_from_params
  from .utils import get_kind_key
  from .utils import group_for_key
  from .utils import reference_property_to_reference
  from .utils import UnprocessedQueryCursor
  from .zkappscale import entity_lock
  from .zkappscale import zktransaction
  from .zkappscale.transaction_manager import TransactionManager
  
  sys.path.append(APPSCALE_PYTHON_APPSERVER)
  from google.appengine.api import api_base_pb
  from google.appengine.api import datastore_errors
  from google.appengine.api.datastore_distributed import _MAX_ACTIONS_PER_TXN
  from google.appengine.api.taskqueue import taskqueue_service_pb
  from google.appengine.api.taskqueue.taskqueue_distributed import\
    TaskQueueServiceStub
  from google.appengine.datastore import appscale_stub_util
  from google.appengine.datastore import datastore_pb
  from google.appengine.datastore import datastore_index
  from google.appengine.datastore import entity_pb
  from google.appengine.datastore import sortable_pb_encoder
  from google.appengine.datastore.datastore_stub_util import IdToCounter
  from google.appengine.datastore.datastore_stub_util import SEQUENTIAL
  from google.appengine.runtime import apiproxy_errors
  from google.appengine.ext import db
  from google.appengine.ext.db.metadata import Namespace
  from google.net.proto.ProtocolBuffer import ProtocolBufferDecodeError
  
  
  class DatastoreDistributed():
    """ AppScale persistent layer for the datastore API. It is the 
        replacement for the AppServers to persist their data into 
        a distributed datastore instead of a flat file.
    """
    # Max number of results for a query
    _MAXIMUM_RESULTS = 10000
  
    # The number of entries looked at when doing a composite query
    # It will keep looking at this size window when getting the result
    _MAX_COMPOSITE_WINDOW = 10000
  
    # Maximum amount of filter and orderings allowed within a query
    _MAX_QUERY_COMPONENTS = 63
  
    # For enabling and disabling range inclusivity
    _ENABLE_INCLUSIVITY = True
    _DISABLE_INCLUSIVITY = False
  
    # Delimiter between app names and namespace and the rest of an entity key
    _NAMESPACE_SEPARATOR = dbconstants.KEY_DELIMITER
  
    # Delimiter between parameters in index keys.
    _SEPARATOR = dbconstants.KEY_DELIMITER
  
    # This is the terminating string for range queries
    _TERM_STRING = dbconstants.TERMINATING_STRING
  
    # Smallest possible value that is considered non-null and indexable.
    MIN_INDEX_VALUE = '\x01'
  
    # When assigning the first allocated ID, give this value
    _FIRST_VALID_ALLOCATED_ID = 1
  
    # The key we use to lock for allocating new IDs
    _ALLOCATE_ROOT_KEY = "__allocate__"
  
    # Number of times to retry acquiring a lock for non transactions.
    NON_TRANS_LOCK_RETRY_COUNT = 5
  
    # How long to wait before retrying to grab a lock
    LOCK_RETRY_TIME = .5
  
    # Maximum number of allowed composite indexes any one application can
    # register.
    _MAX_NUM_INDEXES = dbconstants.MAX_NUMBER_OF_COMPOSITE_INDEXES
  
    # The position of the prop name when splitting an index entry by the
    # delimiter.
    PROP_NAME_IN_SINGLE_PROP_INDEX = 3
  
    # The cassandra index column that stores the reference to the entity.
    INDEX_REFERENCE_COLUMN = 'reference'
  
    # The number of entities to fetch at a time when updating indices.
    BATCH_SIZE = 100
  
    def __init__(self, datastore_batch, transaction_manager, zookeeper=None,
                 log_level=logging.INFO):
      """
         Constructor.
       
       Args:
         datastore_batch: A reference to the batch datastore interface.
         zookeeper: A reference to the zookeeper interface.
      """
      class_name = self.__class__.__name__
      self.logger = logging.getLogger(class_name)
      self.logger.setLevel(log_level)
  
      assert datastore_batch.valid_data_version()
  
      self.logger.info('Starting {}'.format(class_name))
  
      # datastore accessor used by this class to do datastore operations.
      self.datastore_batch = datastore_batch 
  
      # zookeeper instance for accesing ZK functionality.
      self.zookeeper = zookeeper
  
      # Maintain a stub object for each project using transactional tasks.
      self.taskqueue_stubs = {}
  
      # Maintain a scattered allocator for each project.
      self.scattered_allocators = {}
  
      # Maintain a sequential allocator for each project.
      self.sequential_allocators = {}
  
      self
  
  [... the rest is omitted ...]
files: AppDB/appscale/datastore/datastore_distributed.py
files: import array
  import datetime
  import itertools
  import logging
  import md5
  import random
  import sys
  import threading
  import time
  
  import dbconstants
  import helper_functions
  
  from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
  from kazoo.client import KazooState
  from .dbconstants import APP_ENTITY_SCHEMA
  from .dbconstants import ID_KEY_LENGTH
  from .dbconstants import MAX_TX_DURATION
  from .cassandra_env import cassandra_interface
  from .cassandra_env.entity_id_allocator import EntityIDAllocator
  from .cassandra_env.entity_id_allocator import ScatteredAllocator
  from .cassandra_env.utils import deletions_for_entity
  from .cassandra_env.utils import mutations_for_entity
  from .utils import clean_app_id
  from .utils import encode_entity_table_key
  from .utils import encode_index_pb
  from .utils import encode_path_from_filter
  from .utils import get_composite_index_keys
  from .utils import get_entity_key
  from .utils import get_entity_kind
  from .utils import get_index_key_from_params
  from .utils import get_kind_key
  from .utils import group_for_key
  from .utils import reference_property_to_reference
  from .utils import UnprocessedQueryCursor
  from .zkappscale import entity_lock
  from .zkappscale import zktransaction
  from .zkappscale.transaction_manager import TransactionManager
  
  sys.path.append(APPSCALE_PYTHON_APPSERVER)
  from google.appengine.api import api_base_pb
  from google.appengine.api import datastore_errors
  from google.appengine.api.datastore_distributed import _MAX_ACTIONS_PER_TXN
  from google.appengine.api.taskqueue import taskqueue_service_pb
  from google.appengine.api.taskqueue.taskqueue_distributed import\
    TaskQueueServiceStub
  from google.appengine.datastore import appscale_stub_util
  from google.appengine.datastore import datastore_pb
  from google.appengine.datastore import datastore_index
  from google.appengine.datastore import entity_pb
  from google.appengine.datastore import sortable_pb_encoder
  from google.appengine.datastore.datastore_stub_util import IdToCounter
  from google.appengine.datastore.datastore_stub_util import SEQUENTIAL
  from google.appengine.runtime import apiproxy_errors
  from google.appengine.ext import db
  from google.appengine.ext.db.metadata import Namespace
  from google.net.proto.ProtocolBuffer import ProtocolBufferDecodeError
  
  
  class DatastoreDistributed():
    """ AppScale persistent layer for the datastore API. It is the 
        replacement for the AppServers to persist their data into 
        a distributed datastore instead of a flat file.
    """
    # Max number of results for a query
    _MAXIMUM_RESULTS = 10000
  
    # The number of entries looked at when doing a composite query
    # It will keep looking at this size window when getting the result
    _MAX_COMPOSITE_WINDOW = 10000
  
    # Maximum amount of filter and orderings allowed within a query
    _MAX_QUERY_COMPONENTS = 63
  
    # For enabling and disabling range inclusivity
    _ENABLE_INCLUSIVITY = True
    _DISABLE_INCLUSIVITY = False
  
    # Delimiter between app names and namespace and the rest of an entity key
    _NAMESPACE_SEPARATOR = dbconstants.KEY_DELIMITER
  
    # Delimiter between parameters in index keys.
    _SEPARATOR = dbconstants.KEY_DELIMITER
  
    # This is the terminating string for range queries
    _TERM_STRING = dbconstants.TERMINATING_STRING
  
    # Smallest possible value that is considered non-null and indexable.
    MIN_INDEX_VALUE = '\x01'
  
    # When assigning the first allocated ID, give this value
    _FIRST_VALID_ALLOCATED_ID = 1
  
    # The key we use to lock for allocating new IDs
    _ALLOCATE_ROOT_KEY = "__allocate__"
  
    # Number of times to retry acquiring a lock for non transactions.
    NON_TRANS_LOCK_RETRY_COUNT = 5
  
    # How long to wait before retrying to grab a lock
    LOCK_RETRY_TIME = .5
  
    # Maximum number of allowed composite indexes any one application can
    # register.
    _MAX_NUM_INDEXES = dbconstants.MAX_NUMBER_OF_COMPOSITE_INDEXES
  
    # The position of the prop name when splitting an index entry by the
    # delimiter.
    PROP_NAME_IN_SINGLE_PROP_INDEX = 3
  
    # The cassandra index column that stores the reference to the entity.
    INDEX_REFERENCE_COLUMN = 'reference'
  
    # The number of entities to fetch at a time when updating indices.
    BATCH_SIZE = 100
  
    def __init__(self, datastore_batch, transaction_manager, zookeeper=None,
                 log_level=logging.INFO):
      """
         Constructor.
       
       Args:
         datastore_batch: A reference to the batch datastore interface.
         zookeeper: A reference to the zookeeper interface.
      """
      class_name = self.__class__.__name__
      self.logger = logging.getLogger(class_name)
      self.logger.setLevel(log_level)
  
      assert datastore_batch.valid_data_version()
  
      self.logger.info('Starting {}'.format(class_name))
  
      # datastore accessor used by this class to do datastore operations.
      self.datastore_batch = datastore_batch 
  
      # zookeeper instance for accesing ZK functionality.
      self.zookeeper = zookeeper
  
      # Maintain a stub object for each project using transactional tasks.
      self.taskqueue_stubs = {}
  
      # Maintain a scattered allocator for each project.
      self.scattered_allocators = {}
  
      # Maintain a sequential allocator for each project.
      self.sequential_allocators = {}
  
      self
  
  [... the rest is omitted ...]
files: AppDB/appscale/datastore/datastore_distributed.py
files: """ Generates and keeps track of transaction IDs. """
  from __future__ import division
  
  import json
  import logging
  import time
  
  from kazoo.exceptions import KazooException
  from kazoo.exceptions import NodeExistsError
  from kazoo.exceptions import NotEmptyError
  from tornado.ioloop import IOLoop
  
  from appscale.common.async_retrying import retry_children_watch_coroutine
  from .constants import CONTAINER_PREFIX
  from .constants import COUNTER_NODE_PREFIX
  from .constants import MAX_SEQUENCE_COUNTER
  from .constants import OFFSET_NODE
  from .entity_lock import zk_group_path
  from ..dbconstants import BadRequest
  from ..dbconstants import InternalError
  
  logger = logging.getLogger('appscale-datastore')
  
  # Nodes that indicate a cross-group transaction start with this string.
  XG_PREFIX = 'xg'
  
  
  class ProjectTransactionManager(object):
    """ Generates and keeps track of transaction IDs for a project. """
    def __init__(self, project_id, zk_client):
      """ Creates a new ProjectTransactionManager.
  
      Args:
        project_id: A string specifying a project ID.
      """
      self.project_id = project_id
      self.zk_client = zk_client
  
      self._project_node = '/appscale/apps/{}'.format(self.project_id)
  
      # Allows users to manually modify transaction IDs after a binary migration.
      self._txid_manual_offset = 0
      self._offset_node = '/'.join([self._project_node, OFFSET_NODE])
      self.zk_client.DataWatch(self._offset_node, self._update_offset)
  
      # Allows the manager to use a new container after one is exhausted.
      self._txid_automatic_offset = 0
      self._counter_path = '/'.join([self._project_node, CONTAINER_PREFIX])
      self.zk_client.ensure_path(self._project_node)
      self.zk_client.ensure_path(self._counter_path)
      self.zk_client.ChildrenWatch(self._project_node, self._update_project)
  
      # Containers that do not need to be checked for open transactions.
      self._inactive_containers = set()
  
    def create_transaction_id(self, xg):
      """ Generates a new transaction ID.
  
      Args:
        xg: A boolean indicating a cross-group transaction.
      Returns:
        An integer specifying the created transaction ID.
      Raises:
        InternalError if unable to create a new transaction ID.
      """
      current_time = time.time()
      counter_path_prefix = '/'.join([self._counter_path, COUNTER_NODE_PREFIX])
      try:
        new_path = self.zk_client.create(
          counter_path_prefix, value=str(current_time), sequence=True)
      except KazooException:
        message = 'Unable to create new transaction ID'
        logger.exception(message)
        raise InternalError(message)
  
      counter = int(new_path.split('/')[-1].lstrip(COUNTER_NODE_PREFIX))
  
      if counter < 0:
        logger.debug('Removing invalid counter')
        self._delete_counter(new_path)
        self._update_auto_offset()
        return self.create_transaction_id(xg)
  
      txid = self._txid_manual_offset + self._txid_automatic_offset + counter
  
      if txid == 0:
        self._delete_counter(new_path)
        return self.create_transaction_id(xg)
  
      if xg:
        xg_path = '/'.join([new_path, XG_PREFIX])
        try:
          self.zk_client.create(xg_path, value=str(current_time))
        except KazooException:
          message = 'Unable to create new cross-group transaction ID'
          logger.exception(message)
          raise InternalError(message)
  
      self._last_txid_created = txid
      return txid
  
    def delete_transaction_id(self, txid):
      """ Removes a transaction ID from the list of active transactions.
  
      Args:
        txid: An integer specifying a transaction ID.
      """
      path = self._txid_to_path(txid)
      self._delete_counter(path)
  
    def get_open_transactions(self):
      """ Fetches a list of active transactions.
  
      Returns:
        A list of integers specifying transaction IDs.
      Raises:
        InternalError if unable to fetch list of transaction IDs.
      """
      txids = []
      active_containers = self._active_containers()
      for index, container in enumerate(active_containers):
        container_name = container.split('/')[-1]
        container_count = int(container_name[len(CONTAINER_PREFIX):] or 1)
        container_size = MAX_SEQUENCE_COUNTER + 1
        auto_offset = (container_count - 1) * container_size
        offset = self._txid_manual_offset + auto_offset
  
        try:
          paths = self.zk_client.get_children(container)
        except KazooException:
          message = 'Unable to fetch list of counters'
          logger.exception(message)
          raise InternalError(message)
  
        counter_nodes = [path.split('/')[-1] for path in paths]
        txids.extend([offset + int(node.lstrip(COUNTER_NODE_PREFIX))
                      for node in counter_nodes])
  
        # If there are no counters left in an old container, mark it inactive.
        if not counter_nodes and index < len(active_containers) - 1:
          self._inactive_containers.add(container_name)
  
      return txids
  
    def set_groups(self, txid, groups):
      """ Defines which groups will be involved in a transaction.
  
      Args:
        txid: An integer specifying a transaction ID.
        groups: An iterable of entity group Reference objects.
      """
      txid_path = self._txid_to_path(txid)
      groups_path = '/'.join([txid_path, 'groups'])
      encoded_groups = [zk_group_path(group) for group in groups]
      try:
        self.zk_client.create(groups_path, value=json.dumps(encoded_groups))
      except KazooException:
        message = 'Unable to set lock list for transaction'
        logger.exception(message)
        raise InternalError(message)
  
   
  
  [... the rest is omitted ...]
files: AppDB/appscale/datastore/zkappscale/transaction_manager.py
files: """ Generates and keeps track of transaction IDs. """
  from __future__ import division
  
  import logging
  import time
  
  from kazoo.exceptions import KazooException
  from kazoo.exceptions import NodeExistsError
  from kazoo.exceptions import NotEmptyError
  from tornado.ioloop import IOLoop
  
  from appscale.common.async_retrying import retry_children_watch_coroutine
  from .constants import CONTAINER_PREFIX
  from .constants import COUNTER_NODE_PREFIX
  from .constants import MAX_SEQUENCE_COUNTER
  from .constants import OFFSET_NODE
  from ..dbconstants import BadRequest
  from ..dbconstants import InternalError
  
  logger = logging.getLogger('appscale-datastore')
  
  # Nodes that indicate a cross-group transaction start with this string.
  XG_PREFIX = 'xg'
  
  
  class ProjectTransactionManager(object):
    """ Generates and keeps track of transaction IDs for a project. """
    def __init__(self, project_id, zk_client):
      """ Creates a new ProjectTransactionManager.
  
      Args:
        project_id: A string specifying a project ID.
      """
      self.project_id = project_id
      self.zk_client = zk_client
  
      self._project_node = '/appscale/apps/{}'.format(self.project_id)
  
      # Allows users to manually modify transaction IDs after a binary migration.
      self._txid_manual_offset = 0
      self._offset_node = '/'.join([self._project_node, OFFSET_NODE])
      self.zk_client.DataWatch(self._offset_node, self._update_offset)
  
      # Allows the manager to use a new container after one is exhausted.
      self._txid_automatic_offset = 0
      self._counter_path = '/'.join([self._project_node, CONTAINER_PREFIX])
      self.zk_client.ensure_path(self._project_node)
      self.zk_client.ensure_path(self._counter_path)
      self.zk_client.ChildrenWatch(self._project_node, self._update_project)
  
      # Containers that do not need to be checked for open transactions.
      self._inactive_containers = set()
  
    def create_transaction_id(self, xg):
      """ Generates a new transaction ID.
  
      Args:
        xg: A boolean indicating a cross-group transaction.
      Returns:
        An integer specifying the created transaction ID.
      Raises:
        InternalError if unable to create a new transaction ID.
      """
      current_time = time.time()
      counter_path_prefix = '/'.join([self._counter_path, COUNTER_NODE_PREFIX])
      try:
        new_path = self.zk_client.create(
          counter_path_prefix, value=str(current_time), sequence=True)
      except KazooException:
        message = 'Unable to create new transaction ID'
        logger.exception(message)
        raise InternalError(message)
  
      counter = int(new_path.split('/')[-1].lstrip(COUNTER_NODE_PREFIX))
  
      if counter < 0:
        logger.debug('Removing invalid counter')
        self._delete_counter(new_path)
        self._update_auto_offset()
        return self.create_transaction_id(xg)
  
      txid = self._txid_manual_offset + self._txid_automatic_offset + counter
  
      if txid == 0:
        self._delete_counter(new_path)
        return self.create_transaction_id(xg)
  
      if xg:
        xg_path = '/'.join([new_path, XG_PREFIX])
        try:
          self.zk_client.create(xg_path, value=str(current_time))
        except KazooException:
          message = 'Unable to create new cross-group transaction ID'
          logger.exception(message)
          raise InternalError(message)
  
      self._last_txid_created = txid
      return txid
  
    def delete_transaction_id(self, txid):
      """ Removes a transaction ID from the list of active transactions.
  
      Args:
        txid: An integer specifying a transaction ID.
      """
      corrected_counter = txid - self._txid_manual_offset
  
      # The number of counters a container can store (including 0).
      container_size = MAX_SEQUENCE_COUNTER + 1
  
      container_count = int(corrected_counter / container_size) + 1
      container_suffix = '' if container_count == 1 else str(container_count)
      container_name = CONTAINER_PREFIX + container_suffix
      container_path = '/'.join([self._project_node, container_name])
  
      counter_value = corrected_counter % container_size
      node_name = COUNTER_NODE_PREFIX + str(counter_value).zfill(10)
      full_path = '/'.join([container_path, node_name])
      self._delete_counter(full_path)
  
    def get_open_transactions(self):
      """ Fetches a list of active transactions.
  
      Returns:
        A list of integers specifying transaction IDs.
      Raises:
        InternalError if unable to fetch list of transaction IDs.
      """
      txids = []
      active_containers = self._active_containers()
      for index, container in enumerate(active_containers):
        container_name = container.split('/')[-1]
        container_count = int(container_name[len(CONTAINER_PREFIX):] or 1)
        container_size = MAX_SEQUENCE_COUNTER + 1
        auto_offset = (container_count - 1) * container_size
        offset = self._txid_manual_offset + auto_offset
  
        try:
          paths = self.zk_client.get_children(container)
        except KazooException:
          message = 'Unable to fetch list of counters'
          logger.exception(message)
          raise InternalError(message)
  
        counter_nodes = [path.split('/')[-1] for path in paths]
        txids.extend([offset + int(node.lstrip(COUNTER_NODE_PREFIX))
                      for node in counter_nodes])
  
        # If there are no counters left in an old container, mark it inactive.
        if not counter_nodes and index < len(active_containers) - 1:
          self._inactive_containers.add(container_name)
  
      return txids
  
    def _delete_counter(self, path):
      """ Removes a counter node.
  
      Args:
       
  
  [... the rest is omitted ...]
files: AppDB/appscale/datastore/zkappscale/transaction_manager.py
files: #!/usr/bin/env python
  # Programmer: Navraj Chohan <nlake44@gmail.com>
  
  import datetime
  import random
  import sys
  import unittest
  
  from appscale.common import appscale_info
  from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
  from appscale.datastore import dbconstants
  from appscale.datastore import utils
  from appscale.datastore.datastore_distributed import DatastoreDistributed
  from appscale.datastore.dbconstants import APP_ENTITY_SCHEMA
  from appscale.datastore.dbconstants import JOURNAL_SCHEMA
  from appscale.datastore.dbconstants import TOMBSTONE
  from appscale.datastore.cassandra_env.entity_id_allocator import\
    ScatteredAllocator
  
  from appscale.datastore.cassandra_env.cassandra_interface import (
    DatastoreProxy)
  from appscale.datastore.cassandra_env.utils import deletions_for_entity
  from appscale.datastore.cassandra_env.utils import index_deletions
  from appscale.datastore.cassandra_env.utils import mutations_for_entity
  
  from appscale.datastore.utils import (
    get_entity_key,
    get_entity_kind,
    get_index_key_from_params,
    get_index_kv_from_tuple,
    get_kind_key
  )
  
  from appscale.datastore.zkappscale.entity_lock import EntityLock
  from appscale.datastore.zkappscale.zktransaction import ZKTransactionException
  from cassandra.cluster import Cluster
  from flexmock import flexmock
  
  sys.path.append(APPSCALE_PYTHON_APPSERVER)
  from google.appengine.api import api_base_pb
  from google.appengine.datastore import entity_pb
  from google.appengine.datastore import datastore_pb
  from google.appengine.ext import db
  
  
  class Item(db.Model):
    name = db.StringProperty(required = True)
  
  
  class TestDatastoreServer(unittest.TestCase):
    """
    A set of test cases for the datastore server (datastore server v2)
    """
    BASIC_ENTITY = ['guestbook', 'Greeting', 'foo', 'content', 'hello world']
  
    def get_zookeeper(self):
      zk_handle = flexmock(handler=flexmock(event_object=lambda: None,
                                            sleep_func=lambda: None,
                                            lock_object=lambda: None))
      zk_handle.should_receive('add_listener')
      zookeeper = flexmock(handle=zk_handle)
      zookeeper.should_receive("acquire_lock").and_return(True)
      zookeeper.should_receive("release_lock").and_return(True)
      zookeeper.should_receive("increment_and_get_counter").and_return(0,1000)
      return zookeeper
  
    def test_get_entity_kind(self):
      zk_client = flexmock()
      zk_client.should_receive('add_listener')
  
      zookeeper = flexmock(handle=zk_client)
      db_batch = flexmock()
      db_batch.should_receive('valid_data_version').and_return(True)
      item = Item(name="Bob", _app="hello")
      key = db.model_to_protobuf(item)
      self.assertEquals(get_entity_kind(key), "Item")
  
    def test_kind_key(self):
      zk_client = flexmock()
      zk_client.should_receive('add_listener')
  
      zookeeper = flexmock(handle=zk_client)
      db_batch = flexmock()
      db_batch.should_receive('valid_data_version').and_return(True)
      item = Item(name="Dyan", _app="hello")
      key = db.model_to_protobuf(item)
      self.assertEquals(get_kind_key("howdy", key.key().path()), "howdy\x00Item\x01Item:0000000000\x01")
  
      item1 = Item(key_name="Bob", name="Bob", _app="hello")
      key = db.model_to_protobuf(item1)
      self.assertEquals(get_kind_key("howdy", key.key().path()), "howdy\x00Item\x01Item:Bob\x01")
     
      item2 = Item(key_name="Frank", name="Frank", _app="hello", parent = item1)
      key = db.model_to_protobuf(item2)
      self.assertEquals(get_kind_key("howdy", key.key().path()),
             "howdy\x00Item\x01Item:Bob\x01Item:Frank\x01")
  
    def test_get_entity_key(self):
      zk_client = flexmock()
      zk_client.should_receive('add_listener')
  
      zookeeper = flexmock(handle=zk_client)
      db_batch = flexmock()
      db_batch.should_receive('valid_data_version').and_return(True)
      item = Item(key_name="Bob", name="Bob", _app="hello")
      key = db.model_to_protobuf(item)
      self.assertEquals(str(get_entity_key("howdy", key.key().path())), "howdy\x00Item:Bob\x01")
  
    def test_validate_key(self):
      zk_client = flexmock()
      zk_client.should_receive('add_listener')
  
      zookeeper = flexmock(handle=zk_client)
      db_batch = flexmock()
      db_batch.should_receive('valid_data_version').and_return(True)
      transaction_manager = flexmock()
      dd = DatastoreDistributed(db_batch, transaction_manager, zookeeper)
      item = Item(key_name="Bob", name="Bob", _app="hello")
      key = db.model_to_protobuf(item)
      dd.validate_key(key.key())
  
    def test_get_table_prefix(self):
      db_batch = flexmock()
      db_batch.should_receive('valid_data_version').and_return(True)
      db_batch.should_receive("batch_put_entity").and_return(None)
  
      zk_client = flexmock()
      zk_client.should_receive('add_listener')
  
      zookeeper = flexmock(handle=zk_client)
      zookeeper.should_receive("acquire_lock
  
  [... the rest is omitted ...]
files: AppDB/test/unit/test_datastore_server.py
files: #!/usr/bin/env python
  # Programmer: Navraj Chohan <nlake44@gmail.com>
  
  import datetime
  import random
  import sys
  import unittest
  
  from appscale.common import appscale_info
  from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
  from appscale.datastore import dbconstants
  from appscale.datastore import utils
  from appscale.datastore.datastore_distributed import DatastoreDistributed
  from appscale.datastore.dbconstants import APP_ENTITY_SCHEMA
  from appscale.datastore.dbconstants import JOURNAL_SCHEMA
  from appscale.datastore.dbconstants import TOMBSTONE
  from appscale.datastore.cassandra_env.entity_id_allocator import\
    ScatteredAllocator
  
  from appscale.datastore.cassandra_env.cassandra_interface import (
    DatastoreProxy)
  from appscale.datastore.cassandra_env.utils import deletions_for_entity
  from appscale.datastore.cassandra_env.utils import index_deletions
  from appscale.datastore.cassandra_env.utils import mutations_for_entity
  
  from appscale.datastore.utils import (
    get_entity_key,
    get_entity_kind,
    get_index_key_from_params,
    get_index_kv_from_tuple,
    get_kind_key
  )
  
  from appscale.datastore.zkappscale.entity_lock import EntityLock
  from appscale.datastore.zkappscale.zktransaction import ZKTransactionException
  from cassandra.cluster import Cluster
  from flexmock import flexmock
  
  sys.path.append(APPSCALE_PYTHON_APPSERVER)
  from google.appengine.api import api_base_pb
  from google.appengine.datastore import entity_pb
  from google.appengine.datastore import datastore_pb
  from google.appengine.ext import db
  
  
  class Item(db.Model):
    name = db.StringProperty(required = True)
  
  
  class TestDatastoreServer(unittest.TestCase):
    """
    A set of test cases for the datastore server (datastore server v2)
    """
    BASIC_ENTITY = ['guestbook', 'Greeting', 'foo', 'content', 'hello world']
  
    def get_zookeeper(self):
      zk_handle = flexmock(handler=flexmock(event_object=lambda: None,
                                            sleep_func=lambda: None,
                                            lock_object=lambda: None))
      zk_handle.should_receive('add_listener')
      zookeeper = flexmock(handle=zk_handle)
      zookeeper.should_receive("acquire_lock").and_return(True)
      zookeeper.should_receive("release_lock").and_return(True)
      zookeeper.should_receive("increment_and_get_counter").and_return(0,1000)
      return zookeeper
  
    def test_get_entity_kind(self):
      zk_client = flexmock()
      zk_client.should_receive('add_listener')
  
      zookeeper = flexmock(handle=zk_client)
      db_batch = flexmock()
      db_batch.should_receive('valid_data_version').and_return(True)
      item = Item(name="Bob", _app="hello")
      key = db.model_to_protobuf(item)
      self.assertEquals(get_entity_kind(key), "Item")
  
    def test_kind_key(self):
      zk_client = flexmock()
      zk_client.should_receive('add_listener')
  
      zookeeper = flexmock(handle=zk_client)
      db_batch = flexmock()
      db_batch.should_receive('valid_data_version').and_return(True)
      item = Item(name="Dyan", _app="hello")
      key = db.model_to_protobuf(item)
      self.assertEquals(get_kind_key("howdy", key.key().path()), "howdy\x00Item\x01Item:0000000000\x01")
  
      item1 = Item(key_name="Bob", name="Bob", _app="hello")
      key = db.model_to_protobuf(item1)
      self.assertEquals(get_kind_key("howdy", key.key().path()), "howdy\x00Item\x01Item:Bob\x01")
     
      item2 = Item(key_name="Frank", name="Frank", _app="hello", parent = item1)
      key = db.model_to_protobuf(item2)
      self.assertEquals(get_kind_key("howdy", key.key().path()),
             "howdy\x00Item\x01Item:Bob\x01Item:Frank\x01")
  
    def test_get_entity_key(self):
      zk_client = flexmock()
      zk_client.should_receive('add_listener')
  
      zookeeper = flexmock(handle=zk_client)
      db_batch = flexmock()
      db_batch.should_receive('valid_data_version').and_return(True)
      item = Item(key_name="Bob", name="Bob", _app="hello")
      key = db.model_to_protobuf(item)
      self.assertEquals(str(get_entity_key("howdy", key.key().path())), "howdy\x00Item:Bob\x01")
  
    def test_validate_key(self):
      zk_client = flexmock()
      zk_client.should_receive('add_listener')
  
      zookeeper = flexmock(handle=zk_client)
      db_batch = flexmock()
      db_batch.should_receive('valid_data_version').and_return(True)
      transaction_manager = flexmock()
      dd = DatastoreDistributed(db_batch, transaction_manager, zookeeper)
      item = Item(key_name="Bob", name="Bob", _app="hello")
      key = db.model_to_protobuf(item)
      dd.validate_key(key.key())
  
    def test_get_table_prefix(self):
      db_batch = flexmock()
      db_batch.should_receive('valid_data_version').and_return(True)
      db_batch.should_receive("batch_put_entity").and_return(None)
  
      zk_client = flexmock()
      zk_client.should_receive('add_listener')
  
      zookeeper = flexmock(handle=zk_client)
      zookeeper.should_receive("acquire_lock
  
  [... the rest is omitted ...]
files: AppDB/test/unit/test_datastore_server.py