files.new_contents: import array
  import datetime
  import itertools
  import logging
  import md5
  import random
  import sys
  import threading
  import time
  
  import dbconstants
  import helper_functions
  
  from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
  from kazoo.client import KazooState
  from .dbconstants import APP_ENTITY_SCHEMA
  from .dbconstants import ID_KEY_LENGTH
  from .dbconstants import MAX_TX_DURATION
  from .cassandra_env import cassandra_interface
  from .cassandra_env.entity_id_allocator import EntityIDAllocator
  from .cassandra_env.entity_id_allocator import ScatteredAllocator
  from .cassandra_env.utils import deletions_for_entity
  from .cassandra_env.utils import mutations_for_entity
  from .utils import clean_app_id
  from .utils import encode_entity_table_key
  from .utils import encode_index_pb
  from .utils import encode_path_from_filter
  from .utils import get_composite_index_keys
  from .utils import get_entity_key
  from .utils import get_entity_kind
  from .utils import get_index_key_from_params
  from .utils import get_kind_key
  from .utils import group_for_key
  from .utils import reference_property_to_reference
  from .utils import UnprocessedQueryCursor
  from .zkappscale import entity_lock
  from .zkappscale import zktransaction
  from .zkappscale.transaction_manager import TransactionManager
  
  sys.path.append(APPSCALE_PYTHON_APPSERVER)
  from google.appengine.api import api_base_pb
  from google.appengine.api import datastore_errors
  from google.appengine.api.datastore_distributed import _MAX_ACTIONS_PER_TXN
  from google.appengine.api.taskqueue import taskqueue_service_pb
  from google.appengine.api.taskqueue.taskqueue_distributed import\
    TaskQueueServiceStub
  from google.appengine.datastore import appscale_stub_util
  from google.appengine.datastore import datastore_pb
  from google.appengine.datastore import datastore_index
  from google.appengine.datastore import entity_pb
  from google.appengine.datastore import sortable_pb_encoder
  from google.appengine.datastore.datastore_stub_util import IdToCounter
  from google.appengine.datastore.datastore_stub_util import SEQUENTIAL
  from google.appengine.runtime import apiproxy_errors
  from google.appengine.ext import db
  from google.appengine.ext.db.metadata import Namespace
  from google.net.proto.ProtocolBuffer import ProtocolBufferDecodeError
  
  
  class DatastoreDistributed():
    """ AppScale persistent layer for the datastore API. It is the 
        replacement for the AppServers to persist their data into 
        a distributed datastore instead of a flat file.
    """
    # Max number of results for a query
    _MAXIMUM_RESULTS = 10000
  
    # The number of entries looked at when doing a composite query
    # It will keep looking at this size window when getting
  
  [... the rest is omitted ...]
files.new_path: AppDB/appscale/datastore/datastore_distributed.py
files.old_contents: import array
  import datetime
  import itertools
  import logging
  import md5
  import random
  import sys
  import threading
  import time
  
  import dbconstants
  import helper_functions
  
  from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
  from kazoo.client import KazooState
  from .dbconstants import APP_ENTITY_SCHEMA
  from .dbconstants import ID_KEY_LENGTH
  from .dbconstants import MAX_TX_DURATION
  from .cassandra_env import cassandra_interface
  from .cassandra_env.entity_id_allocator import EntityIDAllocator
  from .cassandra_env.entity_id_allocator import ScatteredAllocator
  from .cassandra_env.utils import deletions_for_entity
  from .cassandra_env.utils import mutations_for_entity
  from .utils import clean_app_id
  from .utils import encode_entity_table_key
  from .utils import encode_index_pb
  from .utils import encode_path_from_filter
  from .utils import get_composite_index_keys
  from .utils import get_entity_key
  from .utils import get_entity_kind
  from .utils import get_index_key_from_params
  from .utils import get_kind_key
  from .utils import group_for_key
  from .utils import reference_property_to_reference
  from .utils import UnprocessedQueryCursor
  from .zkappscale import entity_lock
  from .zkappscale import zktransaction
  from .zkappscale.transaction_manager import TransactionManager
  
  sys.path.append(APPSCALE_PYTHON_APPSERVER)
  from google.appengine.api import api_base_pb
  from google.appengine.api import datastore_errors
  from google.appengine.api.datastore_distributed import _MAX_ACTIONS_PER_TXN
  from google.appengine.api.taskqueue import taskqueue_service_pb
  from google.appengine.api.taskqueue.taskqueue_distributed import\
    TaskQueueServiceStub
  from google.appengine.datastore import appscale_stub_util
  from google.appengine.datastore import datastore_pb
  from google.appengine.datastore import datastore_index
  from google.appengine.datastore import entity_pb
  from google.appengine.datastore import sortable_pb_encoder
  from google.appengine.datastore.datastore_stub_util import IdToCounter
  from google.appengine.datastore.datastore_stub_util import SEQUENTIAL
  from google.appengine.runtime import apiproxy_errors
  from google.appengine.ext import db
  from google.appengine.ext.db.metadata import Namespace
  from google.net.proto.ProtocolBuffer import ProtocolBufferDecodeError
  
  
  class DatastoreDistributed():
    """ AppScale persistent layer for the datastore API. It is the 
        replacement for the AppServers to persist their data into 
        a distributed datastore instead of a flat file.
    """
    # Max number of results for a query
    _MAXIMUM_RESULTS = 10000
  
    # The number of entries looked at when doing a composite query
    # It will keep looking at this size window when getting
  
  [... the rest is omitted ...]
files.old_path: AppDB/appscale/datastore/datastore_distributed.py
files.new_contents: """ Generates and keeps track of transaction IDs. """
  from __future__ import division
  
  import json
  import logging
  import time
  
  from kazoo.exceptions import KazooException
  from kazoo.exceptions import NodeExistsError
  from kazoo.exceptions import NotEmptyError
  from tornado.ioloop import IOLoop
  
  from appscale.common.async_retrying import retry_children_watch_coroutine
  from .constants import CONTAINER_PREFIX
  from .constants import COUNTER_NODE_PREFIX
  from .constants import MAX_SEQUENCE_COUNTER
  from .constants import OFFSET_NODE
  from .entity_lock import zk_group_path
  from ..dbconstants import BadRequest
  from ..dbconstants import InternalError
  
  logger = logging.getLogger('appscale-datastore')
  
  # Nodes that indicate a cross-group transaction start with this string.
  XG_PREFIX = 'xg'
  
  
  class ProjectTransactionManager(object):
    """ Generates and keeps track of transaction IDs for a project. """
    def __init__(self, project_id, zk_client):
      """ Creates a new ProjectTransactionManager.
  
      Args:
        project_id: A string specifying a project ID.
      """
      self.project_id = project_id
      self.zk_client = zk_client
  
      self._project_node = '/appscale/apps/{}'.format(self.project_id)
  
      # Allows users to manually modify transaction IDs after a binary migration.
      self._txid_manual_offset = 0
      self._offset_node = '/'.join([self._project_node, OFFSET_NODE])
      self.zk_client.DataWatch(self._offset_node, self._update_offset)
  
      # Allows the manager to use a new container after one is exhausted.
      self._txid_automatic_offset = 0
      self._counter_path = '/'.join([self._project_node, CONTAINER_PREFIX])
      self.zk_client.ensure_path(self._project_node)
      self.zk_client.ensure_path(self._counter_path)
      self.zk_client.ChildrenWatch(self._project_node, self._update_project)
  
      # Containers that do not need to be checked for open transactions.
      self._inactive_containers = set()
  
    def create_transaction_id(self, xg):
      """ Generates a new transaction ID.
  
      Args:
        xg: A boolean indicating a cross-group transaction.
      Returns:
        An integer specifying the created transaction ID.
      Raises:
        InternalError if unable to create a new transaction ID.
      """
      current_time = time.time()
      counter_path_prefix = '/'.join([self._counter_path, COUNTER_NODE_PREFIX])
      try:
        new_path = self.zk_client.create(
          counter_path_prefix, value=str(current_time), sequence=True)
      except KazooException:
        message = 'Unable to create new transaction ID'
        logger.exception(message)
        raise InternalError(message)
  
      counter = int(new_path.split('/')[-1].lstrip(COUNTER_NODE_PREFIX))
  
      if counter < 0:
        logger.debug('Removing invalid counter')
        self._delete_counter(new_path
  
  [... the rest is omitted ...]
files.new_path: AppDB/appscale/datastore/zkappscale/transaction_manager.py
files.old_contents: """ Generates and keeps track of transaction IDs. """
  from __future__ import division
  
  import logging
  import time
  
  from kazoo.exceptions import KazooException
  from kazoo.exceptions import NodeExistsError
  from kazoo.exceptions import NotEmptyError
  from tornado.ioloop import IOLoop
  
  from appscale.common.async_retrying import retry_children_watch_coroutine
  from .constants import CONTAINER_PREFIX
  from .constants import COUNTER_NODE_PREFIX
  from .constants import MAX_SEQUENCE_COUNTER
  from .constants import OFFSET_NODE
  from ..dbconstants import BadRequest
  from ..dbconstants import InternalError
  
  logger = logging.getLogger('appscale-datastore')
  
  # Nodes that indicate a cross-group transaction start with this string.
  XG_PREFIX = 'xg'
  
  
  class ProjectTransactionManager(object):
    """ Generates and keeps track of transaction IDs for a project. """
    def __init__(self, project_id, zk_client):
      """ Creates a new ProjectTransactionManager.
  
      Args:
        project_id: A string specifying a project ID.
      """
      self.project_id = project_id
      self.zk_client = zk_client
  
      self._project_node = '/appscale/apps/{}'.format(self.project_id)
  
      # Allows users to manually modify transaction IDs after a binary migration.
      self._txid_manual_offset = 0
      self._offset_node = '/'.join([self._project_node, OFFSET_NODE])
      self.zk_client.DataWatch(self._offset_node, self._update_offset)
  
      # Allows the manager to use a new container after one is exhausted.
      self._txid_automatic_offset = 0
      self._counter_path = '/'.join([self._project_node, CONTAINER_PREFIX])
      self.zk_client.ensure_path(self._project_node)
      self.zk_client.ensure_path(self._counter_path)
      self.zk_client.ChildrenWatch(self._project_node, self._update_project)
  
      # Containers that do not need to be checked for open transactions.
      self._inactive_containers = set()
  
    def create_transaction_id(self, xg):
      """ Generates a new transaction ID.
  
      Args:
        xg: A boolean indicating a cross-group transaction.
      Returns:
        An integer specifying the created transaction ID.
      Raises:
        InternalError if unable to create a new transaction ID.
      """
      current_time = time.time()
      counter_path_prefix = '/'.join([self._counter_path, COUNTER_NODE_PREFIX])
      try:
        new_path = self.zk_client.create(
          counter_path_prefix, value=str(current_time), sequence=True)
      except KazooException:
        message = 'Unable to create new transaction ID'
        logger.exception(message)
        raise InternalError(message)
  
      counter = int(new_path.split('/')[-1].lstrip(COUNTER_NODE_PREFIX))
  
      if counter < 0:
        logger.debug('Removing invalid counter')
        self._delete_counter(new_path)
        self._update_auto_offset()
        return self.create
  
  [... the rest is omitted ...]
files.old_path: AppDB/appscale/datastore/zkappscale/transaction_manager.py
files.new_contents: #!/usr/bin/env python
  # Programmer: Navraj Chohan <nlake44@gmail.com>
  
  import datetime
  import random
  import sys
  import unittest
  
  from appscale.common import appscale_info
  from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
  from appscale.datastore import dbconstants
  from appscale.datastore import utils
  from appscale.datastore.datastore_distributed import DatastoreDistributed
  from appscale.datastore.dbconstants import APP_ENTITY_SCHEMA
  from appscale.datastore.dbconstants import JOURNAL_SCHEMA
  from appscale.datastore.dbconstants import TOMBSTONE
  from appscale.datastore.cassandra_env.entity_id_allocator import\
    ScatteredAllocator
  
  from appscale.datastore.cassandra_env.cassandra_interface import (
    DatastoreProxy)
  from appscale.datastore.cassandra_env.utils import deletions_for_entity
  from appscale.datastore.cassandra_env.utils import index_deletions
  from appscale.datastore.cassandra_env.utils import mutations_for_entity
  
  from appscale.datastore.utils import (
    get_entity_key,
    get_entity_kind,
    get_index_key_from_params,
    get_index_kv_from_tuple,
    get_kind_key
  )
  
  from appscale.datastore.zkappscale.entity_lock import EntityLock
  from appscale.datastore.zkappscale.zktransaction import ZKTransactionException
  from cassandra.cluster import Cluster
  from flexmock import flexmock
  
  sys.path.append(APPSCALE_PYTHON_APPSERVER)
  from google.appengine.api import api_base_pb
  from google.appengine.datastore import entity_pb
  from google.appengine.datastore import datastore_pb
  from google.appengine.ext import db
  
  
  class Item(db.Model):
    name = db.StringProperty(required = True)
  
  
  class TestDatastoreServer(unittest.TestCase):
    """
    A set of test cases for the datastore server (datastore server v2)
    """
    BASIC_ENTITY = ['guestbook', 'Greeting', 'foo', 'content', 'hello world']
  
    def get_zookeeper(self):
      zk_handle = flexmock(handler=flexmock(event_object=lambda: None,
                                            sleep_func=lambda: None,
                                            lock_object=lambda: None))
      zk_handle.should_receive('add_listener')
      zookeeper = flexmock(handle=zk_handle)
      zookeeper.should_receive("acquire_lock").and_return(True)
      zookeeper.should_receive("release_lock").and_return(True)
      zookeeper.should_receive("increment_and_get_counter").and_return(0,1000)
      return zookeeper
  
    def test_get_entity_kind(self):
      zk_client = flexmock()
      zk_client.should_receive('add_listener')
  
      zookeeper = flexmock(handle=zk_client)
      db_batch = flexmock()
      db_batch.should_receive('valid_data
  
  [... the rest is omitted ...]
files.new_path: AppDB/test/unit/test_datastore_server.py
files.old_contents: #!/usr/bin/env python
  # Programmer: Navraj Chohan <nlake44@gmail.com>
  
  import datetime
  import random
  import sys
  import unittest
  
  from appscale.common import appscale_info
  from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
  from appscale.datastore import dbconstants
  from appscale.datastore import utils
  from appscale.datastore.datastore_distributed import DatastoreDistributed
  from appscale.datastore.dbconstants import APP_ENTITY_SCHEMA
  from appscale.datastore.dbconstants import JOURNAL_SCHEMA
  from appscale.datastore.dbconstants import TOMBSTONE
  from appscale.datastore.cassandra_env.entity_id_allocator import\
    ScatteredAllocator
  
  from appscale.datastore.cassandra_env.cassandra_interface import (
    DatastoreProxy)
  from appscale.datastore.cassandra_env.utils import deletions_for_entity
  from appscale.datastore.cassandra_env.utils import index_deletions
  from appscale.datastore.cassandra_env.utils import mutations_for_entity
  
  from appscale.datastore.utils import (
    get_entity_key,
    get_entity_kind,
    get_index_key_from_params,
    get_index_kv_from_tuple,
    get_kind_key
  )
  
  from appscale.datastore.zkappscale.entity_lock import EntityLock
  from appscale.datastore.zkappscale.zktransaction import ZKTransactionException
  from cassandra.cluster import Cluster
  from flexmock import flexmock
  
  sys.path.append(APPSCALE_PYTHON_APPSERVER)
  from google.appengine.api import api_base_pb
  from google.appengine.datastore import entity_pb
  from google.appengine.datastore import datastore_pb
  from google.appengine.ext import db
  
  
  class Item(db.Model):
    name = db.StringProperty(required = True)
  
  
  class TestDatastoreServer(unittest.TestCase):
    """
    A set of test cases for the datastore server (datastore server v2)
    """
    BASIC_ENTITY = ['guestbook', 'Greeting', 'foo', 'content', 'hello world']
  
    def get_zookeeper(self):
      zk_handle = flexmock(handler=flexmock(event_object=lambda: None,
                                            sleep_func=lambda: None,
                                            lock_object=lambda: None))
      zk_handle.should_receive('add_listener')
      zookeeper = flexmock(handle=zk_handle)
      zookeeper.should_receive("acquire_lock").and_return(True)
      zookeeper.should_receive("release_lock").and_return(True)
      zookeeper.should_receive("increment_and_get_counter").and_return(0,1000)
      return zookeeper
  
    def test_get_entity_kind(self):
      zk_client = flexmock()
      zk_client.should_receive('add_listener')
  
      zookeeper = flexmock(handle=zk_client)
      db_batch = flexmock()
      db_batch.should_receive('valid_data
  
  [... the rest is omitted ...]
files.old_path: AppDB/test/unit/test_datastore_server.py