files.new_contents: import array
  import datetime
  import itertools
  import logging
  import md5
  import random
  import sys
  import threading
  import time
  
  import dbconstants
  import helper_functions
  
  from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
  from kazoo.client import KazooState
  from .dbconstants import APP_ENTITY_SCHEMA
  from .dbconstants import ID_KEY_LENGTH
  from .dbconstants import MAX_TX_DURATION
  from .dbconstants import TimeoutError
  from .cassandra_env import cassandra_interface
  from .cassandra_env.entity_id_allocator import EntityIDAllocator
  from .cassandra_env.entity_id_allocator import ScatteredAllocator
  from .cassandra_env.utils import deletions_for_entity
  from .cassandra_env.utils import mutations_for_entity
  from .utils import clean_app_id
  from .utils import encode_entity_table_key
  from .utils import encode_index_pb
  from .utils import encode_path_from_filter
  from .utils import get_composite_index_keys
  from .utils import get_entity_key
  from .utils import get_entity_kind
  from .utils import get_index_key_from_params
  from .utils import get_kind_key
  from .utils import group_for_key
  from .utils import reference_property_to_reference
  from .utils import UnprocessedQueryCursor
  from .zkappscale import entity_lock
  from .zkappscale import zktransaction
  from .zkappscale.transaction_manager import TransactionManager
  
  sys.path.append(APPSCALE_PYTHON_APPSERVER)
  from google.appengine.api import api_base_pb
  from google.appengine.api import datastore_errors
  from google.appengine.api.datastore_distributed import _MAX_ACTIONS_PER_TXN
  from google.appengine.api.taskqueue import taskqueue_service_pb
  from google.appengine.api.taskqueue.taskqueue_distributed import\
    TaskQueueServiceStub
  from google.appengine.datastore import appscale_stub_util
  from google.appengine.datastore import datastore_pb
  from google.appengine.datastore import datastore_index
  from google.appengine.datastore import entity_pb
  from google.appengine.datastore import sortable_pb_encoder
  from google.appengine.datastore.datastore
  
  [... the rest is omitted ...]
files.new_path: AppDB/appscale/datastore/datastore_distributed.py
files.old_contents: import array
  import datetime
  import itertools
  import logging
  import md5
  import random
  import sys
  import threading
  import time
  
  import dbconstants
  import helper_functions
  
  from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
  from kazoo.client import KazooState
  from .dbconstants import APP_ENTITY_SCHEMA
  from .dbconstants import ID_KEY_LENGTH
  from .dbconstants import MAX_TX_DURATION
  from .cassandra_env import cassandra_interface
  from .cassandra_env.entity_id_allocator import EntityIDAllocator
  from .cassandra_env.entity_id_allocator import ScatteredAllocator
  from .cassandra_env.utils import deletions_for_entity
  from .cassandra_env.utils import mutations_for_entity
  from .utils import clean_app_id
  from .utils import encode_entity_table_key
  from .utils import encode_index_pb
  from .utils import encode_path_from_filter
  from .utils import get_composite_index_keys
  from .utils import get_entity_key
  from .utils import get_entity_kind
  from .utils import get_index_key_from_params
  from .utils import get_kind_key
  from .utils import group_for_key
  from .utils import reference_property_to_reference
  from .utils import UnprocessedQueryCursor
  from .zkappscale import entity_lock
  from .zkappscale import zktransaction
  from .zkappscale.transaction_manager import TransactionManager
  
  sys.path.append(APPSCALE_PYTHON_APPSERVER)
  from google.appengine.api import api_base_pb
  from google.appengine.api import datastore_errors
  from google.appengine.api.datastore_distributed import _MAX_ACTIONS_PER_TXN
  from google.appengine.api.taskqueue import taskqueue_service_pb
  from google.appengine.api.taskqueue.taskqueue_distributed import\
    TaskQueueServiceStub
  from google.appengine.datastore import appscale_stub_util
  from google.appengine.datastore import datastore_pb
  from google.appengine.datastore import datastore_index
  from google.appengine.datastore import entity_pb
  from google.appengine.datastore import sortable_pb_encoder
  from google.appengine.datastore.datastore_stub_util import IdToCounter
  
  
  [... the rest is omitted ...]
files.old_path: AppDB/appscale/datastore/datastore_distributed.py
files.new_contents: """
   Datastore Constants
  """
  import cassandra.cluster
  
  SECRET_LOCATION = "/etc/appscale/secret.key"
  
  ERROR_DEFAULT = "DB_ERROR:"
  NONEXISTANT_TRANSACTION = "0"
  
  # The datastore's default HTTP port.
  DEFAULT_PORT = 4000
  
  # The lowest character to separate different fields in a row key.
  KEY_DELIMITER = '\x00'
  
  # The character used to separate kinds in an ancestry.
  KIND_SEPARATOR = '\x01'
  
  # HTTP code to indicate that the request is invalid.
  HTTP_BAD_REQUEST = 400
  
  # The length of an ID string. A constant length allows lexicographic ordering.
  ID_KEY_LENGTH = 10
  
  # The character between the kind and the ID/name of an entity.
  ID_SEPARATOR = ":"
  
  # Maximum number of groups allowed in cross group transactions.
  MAX_GROUPS_FOR_XG = 25
  
  # The maximum number of composite indexes an application can have.
  MAX_NUMBER_OF_COMPOSITE_INDEXES = 1000
  
  # The maximum number of seconds a transaction can take. In GAE, transactions
  # "have a maximum duration of 60 seconds with a 10 second idle expiration time
  # after 30 seconds." The 10-second idle check is not yet implemented.
  MAX_TX_DURATION = 60
  
  # A string used to create end keys when doing range queries.
  TERMINATING_STRING = chr(255) * 500
  
  # Tombstone value for soft deletes.
  TOMBSTONE = "APPSCALE_SOFT_DELETE"
  
  TRANSIENT_CASSANDRA_ERRORS = (
    cassandra.Unavailable, cassandra.Timeout, cassandra.CoordinationFailure,
    cassandra.OperationTimedOut, cassandra.cluster.NoHostAvailable)
  
  # The database backends supported by the AppScale datastore.
  VALID_DATASTORES = ['cassandra']
  
  # Table names
  USERS_TABLE = "USERS__"
  JOURNAL_TABLE = "JOURNAL__"
  
  ASC_PROPERTY_TABLE = "ASC_PROPERTY__"
  DSC_PROPERTY_TABLE = "DSC_PROPERTY__"
  COMPOSITE_TABLE = "COMPOSITE_INDEXES__"
  APP_ID_TABLE = "APP_IDS__"
  APP_ENTITY_TABLE = "ENTITIES
  
  [... the rest is omitted ...]
files.new_path: AppDB/appscale/datastore/dbconstants.py
files.old_contents: """
   Datastore Constants
  """
  import cassandra.cluster
  
  SECRET_LOCATION = "/etc/appscale/secret.key"
  
  ERROR_DEFAULT = "DB_ERROR:"
  NONEXISTANT_TRANSACTION = "0"
  
  # The datastore's default HTTP port.
  DEFAULT_PORT = 4000
  
  # The lowest character to separate different fields in a row key.
  KEY_DELIMITER = '\x00'
  
  # The character used to separate kinds in an ancestry.
  KIND_SEPARATOR = '\x01'
  
  # HTTP code to indicate that the request is invalid.
  HTTP_BAD_REQUEST = 400
  
  # The length of an ID string. A constant length allows lexicographic ordering.
  ID_KEY_LENGTH = 10
  
  # The character between the kind and the ID/name of an entity.
  ID_SEPARATOR = ":"
  
  # Maximum number of groups allowed in cross group transactions.
  MAX_GROUPS_FOR_XG = 25
  
  # The maximum number of composite indexes an application can have.
  MAX_NUMBER_OF_COMPOSITE_INDEXES = 1000
  
  # The maximum number of seconds a transaction can take. In GAE, transactions
  # "have a maximum duration of 60 seconds with a 10 second idle expiration time
  # after 30 seconds." The 10-second idle check is not yet implemented.
  MAX_TX_DURATION = 60
  
  # A string used to create end keys when doing range queries.
  TERMINATING_STRING = chr(255) * 500
  
  # Tombstone value for soft deletes.
  TOMBSTONE = "APPSCALE_SOFT_DELETE"
  
  TRANSIENT_CASSANDRA_ERRORS = (
    cassandra.Unavailable, cassandra.Timeout, cassandra.CoordinationFailure,
    cassandra.OperationTimedOut, cassandra.cluster.NoHostAvailable)
  
  # The database backends supported by the AppScale datastore.
  VALID_DATASTORES = ['cassandra']
  
  # Table names
  USERS_TABLE = "USERS__"
  JOURNAL_TABLE = "JOURNAL__"
  
  ASC_PROPERTY_TABLE = "ASC_PROPERTY__"
  DSC_PROPERTY_TABLE = "DSC_PROPERTY__"
  COMPOSITE_TABLE = "COMPOSITE_INDEXES__"
  APP_ID_TABLE = "APP_IDS__"
  APP_ENTITY_TABLE = "ENTITIES
  
  [... the rest is omitted ...]
files.old_path: AppDB/appscale/datastore/dbconstants.py
files.new_contents: # See LICENSE file
  #
  """
  This web service interfaces with the datastore. It takes protocol buffer
  requests from AppServers and responds according to the type of request its
  given (Put, Get, Delete, Query, etc).
  """
  import argparse
  import json
  import logging
  import os
  import sys
  import threading
  import time
  import tornado.httpserver
  import tornado.web
  
  from appscale.common import appscale_info
  from appscale.common.async_retrying import retry_data_watch_coroutine
  from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
  from kazoo.client import KazooState
  from kazoo.exceptions import NodeExistsError
  from tornado import gen
  from tornado.httpclient import AsyncHTTPClient
  from tornado.ioloop import IOLoop
  from tornado.options import options
  from .. import dbconstants
  from ..appscale_datastore_batch import DatastoreFactory
  from ..datastore_distributed import DatastoreDistributed
  from ..utils import (clean_app_id,
                       logger,
                       UnprocessedQueryResult)
  from ..zkappscale import zktransaction
  from ..zkappscale.transaction_manager import TransactionManager
  
  sys.path.append(APPSCALE_PYTHON_APPSERVER)
  from google.appengine.api import api_base_pb
  from google.appengine.api.taskqueue import taskqueue_service_pb
  from google.appengine.datastore import datastore_pb
  from google.appengine.datastore import datastore_v4_pb
  from google.appengine.datastore import entity_pb
  from google.appengine.ext.remote_api import remote_api_pb
  
  # Global for accessing the datastore. An instance of DatastoreDistributed.
  datastore_access = None
  
  # A record of active datastore servers.
  datastore_servers = set()
  
  # The ZooKeeper path where this server registers its availability.
  server_node = None
  
  # ZooKeeper global variable for locking
  zookeeper = None
  
  # Determines whether or not to allow datastore writes. Note: After enabling,
  # datastore processes must be restarted and the groomer must be stopped.
  READ_ONLY = False
  
  # Global stats.
  STATS = {}
  
  # The ZooKeeper path where a list of active datastore servers is stored.
  DATASTORE
  
  [... the rest is omitted ...]
files.new_path: AppDB/appscale/datastore/scripts/datastore.py
files.old_contents: # See LICENSE file
  #
  """
  This web service interfaces with the datastore. It takes protocol buffer
  requests from AppServers and responds according to the type of request its
  given (Put, Get, Delete, Query, etc).
  """
  import argparse
  import json
  import logging
  import os
  import sys
  import threading
  import time
  import tornado.httpserver
  import tornado.web
  
  from appscale.common import appscale_info
  from appscale.common.async_retrying import retry_data_watch_coroutine
  from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
  from kazoo.client import KazooState
  from kazoo.exceptions import NodeExistsError
  from tornado import gen
  from tornado.httpclient import AsyncHTTPClient
  from tornado.ioloop import IOLoop
  from tornado.options import options
  from .. import dbconstants
  from ..appscale_datastore_batch import DatastoreFactory
  from ..datastore_distributed import DatastoreDistributed
  from ..utils import (clean_app_id,
                       logger,
                       UnprocessedQueryResult)
  from ..zkappscale import zktransaction
  from ..zkappscale.transaction_manager import TransactionManager
  
  sys.path.append(APPSCALE_PYTHON_APPSERVER)
  from google.appengine.api import api_base_pb
  from google.appengine.api.taskqueue import taskqueue_service_pb
  from google.appengine.datastore import datastore_pb
  from google.appengine.datastore import datastore_v4_pb
  from google.appengine.datastore import entity_pb
  from google.appengine.ext.remote_api import remote_api_pb
  
  # Global for accessing the datastore. An instance of DatastoreDistributed.
  datastore_access = None
  
  # A record of active datastore servers.
  datastore_servers = set()
  
  # The ZooKeeper path where this server registers its availability.
  server_node = None
  
  # ZooKeeper global variable for locking
  zookeeper = None
  
  # Determines whether or not to allow datastore writes. Note: After enabling,
  # datastore processes must be restarted and the groomer must be stopped.
  READ_ONLY = False
  
  # Global stats.
  STATS = {}
  
  # The ZooKeeper path where a list of active datastore servers is stored.
  DATASTORE
  
  [... the rest is omitted ...]
files.old_path: AppDB/appscale/datastore/scripts/datastore.py
files.new_contents: import base64
  import uuid
  
  from kazoo.exceptions import (
    CancelledError,
    KazooException,
    LockTimeout,
    NoNodeError,
    NotEmptyError
  )
  
  from kazoo.retry import (
    ForceRetryError,
    KazooRetry,
    RetryFailedError
  )
  
  # The ZooKeeper node that contains lock entries for an entity group.
  LOCK_PATH_TEMPLATE = u'/appscale/apps/{project}/locks/{namespace}/{group}'
  
  # The number of seconds to wait for a lock before raising a timeout error.
  LOCK_TIMEOUT = 10
  
  
  def zk_group_path(key):
    """ Retrieve the ZooKeeper lock path for a given entity key.
  
    Args:
      key: An entity Reference object.
    Returns:
      A string containing the location of a ZooKeeper path.
    """
    project = key.app().decode('utf-8')
    if key.name_space():
      namespace = key.name_space().decode('utf-8')
    else:
      namespace = u':default'
  
    first_element = key.path().element(0)
    kind = first_element.type().decode('utf-8')
  
    # Differentiate between types of identifiers.
    if first_element.has_id():
      group = u'{}:{}'.format(kind, first_element.id())
    else:
      # Kazoo does not accept certain characters (eg. newlines) that Cloud
      # Datastore allows.
      encoded_id = base64.b64encode(first_element.name())
      group = u'{}::{}'.format(kind, encoded_id.decode('utf-8').rstrip('='))
  
    return LOCK_PATH_TEMPLATE.format(project=project, namespace=namespace,
                                     group=group)
  
  
  class EntityLock(object):
    """ A ZooKeeper-based entity lock that allows test-and-set operations.
  
    This is based on kazoo's lock recipe, and has been modified to lock multiple
    entity groups. This lock is not re-entrant. Repeated calls after already
    acquired will block.
    """
    _NODE_NAME = '__lock__'
  
    def __init__(self, client, keys, txid=None):
      """ Create an entity lock.
  
      Args:
        client: A kazoo client.
        keys: A list of entity Reference objects.
        txid: An integer specifying the transaction ID.
  
  
  [... the rest is omitted ...]
files.new_path: AppDB/appscale/datastore/zkappscale/entity_lock.py
files.old_contents: import base64
  import uuid
  
  from kazoo.exceptions import (
    CancelledError,
    KazooException,
    LockTimeout,
    NoNodeError,
    NotEmptyError
  )
  
  from kazoo.retry import (
    ForceRetryError,
    KazooRetry,
    RetryFailedError
  )
  
  # The ZooKeeper node that contains lock entries for an entity group.
  LOCK_PATH_TEMPLATE = u'/appscale/apps/{project}/locks/{namespace}/{group}'
  
  # The number of seconds to wait for a lock before raising a timeout error.
  LOCK_TIMEOUT = 10
  
  
  def zk_group_path(key):
    """ Retrieve the ZooKeeper lock path for a given entity key.
  
    Args:
      key: An entity Reference object.
    Returns:
      A string containing the location of a ZooKeeper path.
    """
    project = key.app().decode('utf-8')
    if key.name_space():
      namespace = key.name_space().decode('utf-8')
    else:
      namespace = u':default'
  
    first_element = key.path().element(0)
    kind = first_element.type().decode('utf-8')
  
    # Differentiate between types of identifiers.
    if first_element.has_id():
      group = u'{}:{}'.format(kind, first_element.id())
    else:
      # Kazoo does not accept certain characters (eg. newlines) that Cloud
      # Datastore allows.
      encoded_id = base64.b64encode(first_element.name())
      group = u'{}::{}'.format(kind, encoded_id.decode('utf-8').rstrip('='))
  
    return LOCK_PATH_TEMPLATE.format(project=project, namespace=namespace,
                                     group=group)
  
  
  class EntityLock(object):
    """ A ZooKeeper-based entity lock that allows test-and-set operations.
  
    This is based on kazoo's lock recipe, and has been modified to lock multiple
    entity groups. This lock is not re-entrant. Repeated calls after already
    acquired will block.
    """
    _NODE_NAME = '__lock__'
  
    def __init__(self, client, keys, txid=None):
      """ Create an entity lock.
  
      Args:
        client: A kazoo client.
        keys: A list of entity Reference objects.
        txid: An integer specifying the transaction ID.
  
  
  [... the rest is omitted ...]
files.old_path: AppDB/appscale/datastore/zkappscale/entity_lock.py