files.new_contents: import array
  import datetime
  import itertools
  import logging
  import md5
  import sys
  import uuid
  
  from tornado import gen
  from tornado.ioloop import IOLoop
  
  from appscale.datastore import dbconstants, helper_functions
  
  from appscale.common.datastore_index import DatastoreIndex, merge_indexes
  from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
  from kazoo.client import KazooState
  from appscale.datastore.dbconstants import (
    APP_ENTITY_SCHEMA, BadRequest, ID_KEY_LENGTH, InternalError, MAX_TX_DURATION,
    Timeout
  )
  from appscale.datastore.cassandra_env.cassandra_interface import (
    batch_size, LARGE_BATCH_THRESHOLD)
  from appscale.datastore.cassandra_env.entity_id_allocator import EntityIDAllocator
  from appscale.datastore.cassandra_env.entity_id_allocator import ScatteredAllocator
  from appscale.datastore.cassandra_env.large_batch import BatchNotApplied
  from appscale.datastore.cassandra_env.utils import deletions_for_entity
  from appscale.datastore.cassandra_env.utils import mutations_for_entity
  from appscale.datastore.index_manager import IndexInaccessible
  from appscale.datastore.taskqueue_client import EnqueueError, TaskQueueClient
  from appscale.datastore.utils import _FindIndexToUse
  from appscale.datastore.utils import clean_app_id
  from appscale.datastore.utils import decode_path
  from appscale.datastore.utils import encode_entity_table_key
  from appscale.datastore.utils import encode_index_pb
  from appscale.datastore.utils import encode_path_from_filter
  from appscale.datastore.utils import get_composite_index_keys
  from appscale.datastore.utils import get_entity_key
  from appscale.datastore.utils import get_entity_kind
  from appscale.datastore.utils import get_index_key_from_params
  from appscale.datastore.utils import get_kind_key
  from appscale.datastore.utils import group_for_key
  from appscale.datastore.utils import kind_from_encoded_key
  from appscale.datastore.utils import reference_property_to_reference
  from appscale.datastore.utils import UnprocessedQueryCursor
  from appscale.datastore.range_iterator import RangeExhausted, RangeIterator
  from appscale.datastore.zkappscale import entity_lock
  from appscale.datastore.zkappscale import zktransaction
  
  sys.path.append(APPSCALE_PYTHON_APPSERVER)
  from google.appengine.api import datastore_errors
  from google.appengine.api.datastore_distributed import _MAX_ACTIONS_PER_TXN
  from google.appengine.datastore import appscale_stub_util
  from google.appengine.datastore import datastore_pb
  from google.appengine.datastore import datastore_index
  from google.appengine.datastore import entity_pb
  from google.appengine.datastore import sortable_pb_encoder
  from google.app
  
  [... the rest is omitted ...]
files.new_path: AppDB/appscale/datastore/datastore_distributed.py
files.old_contents: import array
  import datetime
  import itertools
  import logging
  import md5
  import sys
  import uuid
  
  from tornado import gen
  from tornado.ioloop import IOLoop
  
  from appscale.datastore import dbconstants, helper_functions
  
  from appscale.common.datastore_index import DatastoreIndex, merge_indexes
  from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
  from kazoo.client import KazooState
  from appscale.datastore.dbconstants import (
    APP_ENTITY_SCHEMA, BadRequest, ID_KEY_LENGTH, InternalError, MAX_TX_DURATION,
    Timeout
  )
  from appscale.datastore.cassandra_env.cassandra_interface import (
    batch_size, LARGE_BATCH_THRESHOLD)
  from appscale.datastore.cassandra_env.entity_id_allocator import EntityIDAllocator
  from appscale.datastore.cassandra_env.entity_id_allocator import ScatteredAllocator
  from appscale.datastore.cassandra_env.large_batch import BatchNotApplied
  from appscale.datastore.cassandra_env.utils import deletions_for_entity
  from appscale.datastore.cassandra_env.utils import mutations_for_entity
  from appscale.datastore.index_manager import IndexInaccessible
  from appscale.datastore.taskqueue_client import EnqueueError, TaskQueueClient
  from appscale.datastore.utils import _FindIndexToUse
  from appscale.datastore.utils import clean_app_id
  from appscale.datastore.utils import decode_path
  from appscale.datastore.utils import encode_entity_table_key
  from appscale.datastore.utils import encode_index_pb
  from appscale.datastore.utils import encode_path_from_filter
  from appscale.datastore.utils import get_composite_index_keys
  from appscale.datastore.utils import get_entity_key
  from appscale.datastore.utils import get_entity_kind
  from appscale.datastore.utils import get_index_key_from_params
  from appscale.datastore.utils import get_kind_key
  from appscale.datastore.utils import group_for_key
  from appscale.datastore.utils import kind_from_encoded_key
  from appscale.datastore.utils import reference_property_to_reference
  from appscale.datastore.utils import UnprocessedQueryCursor
  from appscale.datastore.range_iterator import RangeExhausted, RangeIterator
  from appscale.datastore.zkappscale import entity_lock
  from appscale.datastore.zkappscale import zktransaction
  
  sys.path.append(APPSCALE_PYTHON_APPSERVER)
  from google.appengine.api import datastore_errors
  from google.appengine.api.datastore_distributed import _MAX_ACTIONS_PER_TXN
  from google.appengine.datastore import appscale_stub_util
  from google.appengine.datastore import datastore_pb
  from google.appengine.datastore import datastore_index
  from google.appengine.datastore import entity_pb
  from google.appengine.datastore import sortable_pb_encoder
  from google.app
  
  [... the rest is omitted ...]
files.old_path: AppDB/appscale/datastore/datastore_distributed.py
files.new_contents: # See LICENSE file
  #
  """
  This web service interfaces with the datastore. It takes protocol buffer
  requests from AppServers and responds according to the type of request its
  given (Put, Get, Delete, Query, etc).
  """
  import argparse
  import json
  import logging
  import os
  import sys
  import time
  import tornado.httpserver
  import tornado.web
  
  from appscale.common import appscale_info
  from appscale.common.appscale_info import get_load_balancer_ips
  from appscale.common.async_retrying import retry_data_watch_coroutine
  from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
  from kazoo.client import KazooState
  from kazoo.exceptions import NodeExistsError
  from tornado import gen
  from tornado.httpclient import AsyncHTTPClient
  from tornado.ioloop import IOLoop
  from tornado.options import options
  from .. import dbconstants
  from ..appscale_datastore_batch import DatastoreFactory
  from ..datastore_distributed import DatastoreDistributed
  from ..index_manager import IndexManager
  from ..utils import (clean_app_id,
                       logger,
                       UnprocessedQueryResult)
  from ..zkappscale import zktransaction
  from ..zkappscale.transaction_manager import TransactionManager
  
  sys.path.append(APPSCALE_PYTHON_APPSERVER)
  from google.appengine.api import api_base_pb
  from google.appengine.api.taskqueue import taskqueue_service_pb
  from google.appengine.datastore import datastore_pb
  from google.appengine.datastore import datastore_v4_pb
  from google.appengine.datastore import entity_pb
  from google.appengine.ext.remote_api import remote_api_pb
  from google.net.proto.ProtocolBuffer import ProtocolBufferDecodeError
  
  # Global for accessing the datastore. An instance of DatastoreDistributed.
  datastore_access = None
  
  # A record of active datastore servers.
  datastore_servers = set()
  
  # The ZooKeeper path where this server registers its availability.
  server_node = None
  
  # ZooKeeper global variable for locking
  zookeeper = None
  
  # Determines whether or not to allow datastore writes. Note: After enabling,
  # datastore processes must be restarted and the groomer must be stopped.
  READ_ONLY = False
  
  # Global stats.
  STATS = {}
  
  # The ZooKeeper path where a list of active datastore servers is stored.
  DATASTORE_SERVERS_NODE = '/appscale/datastore/servers'
  
  
  class ClearHandler(tornado.web.RequestHandler):
    """ Defines what to do when the webserver receives a /clear HTTP request. """
    def set_default_headers(self):
      """ Instructs clients to close the connection after each response. """
      self.set_header('Connection', 'close')
  
    @tornado.web.asynchronous
    def post(self):
      """ Handles POST requests for clearing datastore server stats. """
      global STATS
      STATS = {}
      self.write({"message": "Statistics for this server cleared."
  
  [... the rest is omitted ...]
files.new_path: AppDB/appscale/datastore/scripts/datastore.py
files.old_contents: # See LICENSE file
  #
  """
  This web service interfaces with the datastore. It takes protocol buffer
  requests from AppServers and responds according to the type of request its
  given (Put, Get, Delete, Query, etc).
  """
  import argparse
  import json
  import logging
  import os
  import sys
  import time
  import tornado.httpserver
  import tornado.web
  
  from appscale.common import appscale_info
  from appscale.common.appscale_info import get_load_balancer_ips
  from appscale.common.async_retrying import retry_data_watch_coroutine
  from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
  from kazoo.client import KazooState
  from kazoo.exceptions import NodeExistsError
  from tornado import gen
  from tornado.httpclient import AsyncHTTPClient
  from tornado.ioloop import IOLoop
  from tornado.options import options
  from .. import dbconstants
  from ..appscale_datastore_batch import DatastoreFactory
  from ..datastore_distributed import DatastoreDistributed
  from ..index_manager import IndexManager
  from ..utils import (clean_app_id,
                       logger,
                       UnprocessedQueryResult)
  from ..zkappscale import zktransaction
  from ..zkappscale.transaction_manager import TransactionManager
  
  sys.path.append(APPSCALE_PYTHON_APPSERVER)
  from google.appengine.api import api_base_pb
  from google.appengine.api.taskqueue import taskqueue_service_pb
  from google.appengine.datastore import datastore_pb
  from google.appengine.datastore import datastore_v4_pb
  from google.appengine.datastore import entity_pb
  from google.appengine.ext.remote_api import remote_api_pb
  from google.net.proto.ProtocolBuffer import ProtocolBufferDecodeError
  
  # Global for accessing the datastore. An instance of DatastoreDistributed.
  datastore_access = None
  
  # A record of active datastore servers.
  datastore_servers = set()
  
  # The ZooKeeper path where this server registers its availability.
  server_node = None
  
  # ZooKeeper global variable for locking
  zookeeper = None
  
  # Determines whether or not to allow datastore writes. Note: After enabling,
  # datastore processes must be restarted and the groomer must be stopped.
  READ_ONLY = False
  
  # Global stats.
  STATS = {}
  
  # The ZooKeeper path where a list of active datastore servers is stored.
  DATASTORE_SERVERS_NODE = '/appscale/datastore/servers'
  
  
  class ClearHandler(tornado.web.RequestHandler):
    """ Defines what to do when the webserver receives a /clear HTTP request. """
    def set_default_headers(self):
      """ Instructs clients to close the connection after each response. """
      self.set_header('Connection', 'close')
  
    @tornado.web.asynchronous
    def post(self):
      """ Handles POST requests for clearing datastore server stats. """
      global STATS
      STATS = {}
      self.write({"message": "Statistics for this server cleared."
  
  [... the rest is omitted ...]
files.old_path: AppDB/appscale/datastore/scripts/datastore.py
files.new_contents: #!/usr/bin/env python
  # Programmer: Navraj Chohan <nlake44@gmail.com>
  
  import datetime
  import random
  import sys
  import unittest
  
  from tornado import gen, testing
  
  from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
  from appscale.datastore import dbconstants
  from appscale.datastore import utils
  from appscale.datastore.datastore_distributed import DatastoreDistributed
  from appscale.datastore.dbconstants import APP_ENTITY_SCHEMA
  from appscale.datastore.dbconstants import JOURNAL_SCHEMA
  from appscale.datastore.dbconstants import TOMBSTONE
  from appscale.datastore.cassandra_env.entity_id_allocator import\
    ScatteredAllocator
  
  from appscale.datastore.cassandra_env.utils import deletions_for_entity
  from appscale.datastore.cassandra_env.utils import index_deletions
  from appscale.datastore.cassandra_env.utils import mutations_for_entity
  
  from appscale.datastore.utils import (
    encode_index_pb,
    get_entity_key,
    get_entity_kind,
    get_index_key_from_params,
    get_index_kv_from_tuple,
    get_kind_key
  )
  
  from appscale.datastore.zkappscale.entity_lock import EntityLock
  from appscale.datastore.zkappscale.zktransaction import ZKTransactionException
  from flexmock import flexmock
  
  sys.path.append(APPSCALE_PYTHON_APPSERVER)
  from google.appengine.datastore import entity_pb
  from google.appengine.datastore import datastore_pb
  from google.appengine.ext import db
  
  
  class Item(db.Model):
    name = db.StringProperty(required = True)
  
  
  ASYNC_NONE = gen.Future()
  ASYNC_NONE.set_result(None)
  
  
  class TestDatastoreServer(testing.AsyncTestCase):
    """
    A set of test cases for the datastore server (datastore server v2)
    """
    BASIC_ENTITY = ['guestbook', 'Greeting', 'foo', 'content', 'hello world']
  
    def setUp(self, *args, **kwargs):
      super(TestDatastoreServer, self).setUp(*args, **kwargs)
  
  
  
    def get_zookeeper(self):
      zk_handle = flexmock(handler=flexmock(event_object=lambda: None,
                                            sleep_func=lambda: None,
                                            lock_object=lambda: None))
      zk_handle.should_receive('add_listener')
      zookeeper = flexmock(handle=zk_handle)
      zookeeper.should_receive("acquire_lock").and_return(True)
      zookeeper.should_receive("release_lock").and_return(True)
      zookeeper.should_receive("increment_and_get_counter").and_return(0,1000)
      return zookeeper
  
    def test_get_entity_kind(self):
      item = Item(name="Bob", _app="hello")
      key = db.model_to_protobuf(item)
      self.assertEquals(get_entity_kind(key), "Item")
  
    def test
  
  [... the rest is omitted ...]
files.new_path: AppDB/test/unit/test_datastore_server.py
files.old_contents: #!/usr/bin/env python
  # Programmer: Navraj Chohan <nlake44@gmail.com>
  
  import datetime
  import random
  import sys
  import unittest
  
  from tornado import gen, testing
  
  from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
  from appscale.datastore import dbconstants
  from appscale.datastore import utils
  from appscale.datastore.datastore_distributed import DatastoreDistributed
  from appscale.datastore.dbconstants import APP_ENTITY_SCHEMA
  from appscale.datastore.dbconstants import JOURNAL_SCHEMA
  from appscale.datastore.dbconstants import TOMBSTONE
  from appscale.datastore.cassandra_env.entity_id_allocator import\
    ScatteredAllocator
  
  from appscale.datastore.cassandra_env.utils import deletions_for_entity
  from appscale.datastore.cassandra_env.utils import index_deletions
  from appscale.datastore.cassandra_env.utils import mutations_for_entity
  
  from appscale.datastore.utils import (
    encode_index_pb,
    get_entity_key,
    get_entity_kind,
    get_index_key_from_params,
    get_index_kv_from_tuple,
    get_kind_key
  )
  
  from appscale.datastore.zkappscale.entity_lock import EntityLock
  from appscale.datastore.zkappscale.zktransaction import ZKTransactionException
  from flexmock import flexmock
  
  sys.path.append(APPSCALE_PYTHON_APPSERVER)
  from google.appengine.datastore import entity_pb
  from google.appengine.datastore import datastore_pb
  from google.appengine.ext import db
  
  
  class Item(db.Model):
    name = db.StringProperty(required = True)
  
  
  ASYNC_NONE = gen.Future()
  ASYNC_NONE.set_result(None)
  
  
  class TestDatastoreServer(testing.AsyncTestCase):
    """
    A set of test cases for the datastore server (datastore server v2)
    """
    BASIC_ENTITY = ['guestbook', 'Greeting', 'foo', 'content', 'hello world']
  
    def setUp(self, *args, **kwargs):
      super(TestDatastoreServer, self).setUp(*args, **kwargs)
  
  
  
    def get_zookeeper(self):
      zk_handle = flexmock(handler=flexmock(event_object=lambda: None,
                                            sleep_func=lambda: None,
                                            lock_object=lambda: None))
      zk_handle.should_receive('add_listener')
      zookeeper = flexmock(handle=zk_handle)
      zookeeper.should_receive("acquire_lock").and_return(True)
      zookeeper.should_receive("release_lock").and_return(True)
      zookeeper.should_receive("increment_and_get_counter").and_return(0,1000)
      return zookeeper
  
    def test_get_entity_kind(self):
      item = Item(name="Bob", _app="hello")
      key = db.model_to_protobuf(item)
      self.assertEquals(get_entity_kind(key), "Item")
  
    def test
  
  [... the rest is omitted ...]
files.old_path: AppDB/test/unit/test_datastore_server.py