files.new_contents: # Programmer: Navraj Chohan <nlake44@gmail.com>
  
  """
   Cassandra Interface for AppScale
  """
  import cassandra
  import logging
  import sys
  import time
  
  from cassandra.cluster import Cluster
  from cassandra.concurrent import execute_concurrent
  from cassandra.policies import FallthroughRetryPolicy
  from cassandra.policies import RetryPolicy
  from cassandra.query import BatchStatement
  from cassandra.query import ConsistencyLevel
  from cassandra.query import SimpleStatement
  from cassandra.query import ValueSequence
  from .. import dbconstants
  from .. import helper_functions
  from ..dbconstants import AppScaleDBConnectionError
  from ..dbconstants import Operations
  from ..dbconstants import TxnActions
  from ..dbinterface import AppDBInterface
  from ..unpackaged import APPSCALE_LIB_DIR
  from ..utils import clean_app_id
  from ..utils import encode_index_pb
  from ..utils import get_composite_index_keys
  from ..utils import get_composite_indexes_rows
  from ..utils import get_entity_key
  from ..utils import get_entity_kind
  from ..utils import get_index_kv_from_tuple
  from ..utils import get_kind_key
  
  sys.path.append(APPSCALE_LIB_DIR)
  import appscale_info
  
  # The directory Cassandra is installed to.
  CASSANDRA_INSTALL_DIR = '/opt/cassandra'
  
  # Full path for the nodetool binary.
  NODE_TOOL = '{}/cassandra/bin/nodetool'.format(CASSANDRA_INSTALL_DIR)
  
  # The keyspace used for all tables
  KEYSPACE = "Keyspace1"
  
  # Cassandra watch name.
  CASSANDRA_MONIT_WATCH_NAME = "cassandra-9999"
  
  # The number of times to retry connecting to Cassandra.
  INITIAL_CONNECT_RETRIES = 20
  
  # The data layout version that the datastore expects.
  EXPECTED_DATA_VERSION = 1.0
  
  # The metadata key for the data layout version.
  VERSION_INFO_KEY = 'version'
  
  # The metadata key indicating that the database has been primed.
  PRIMED_KEY = 'primed'
  
  # The size in bytes that a batch must be to use the batches table.
  LARGE_BATCH_THRESHOLD = 5 << 10
  
  
  def batch_size(batch):
    """ Calculates the size of a batch.
  
    Args:
      batch: A list of dictionaries representing mutations.
    Returns:
      An integer specifying the size in bytes of the batch.
    """
    size = 0
    for mutation in batch:
      size += len(mutation['key'])
      if 'values' in mutation:
        for value in mutation['values'].values():
          size += len(value)
    return size
  
  
  def deletions_for_entity(entity, composite_indices=()):
    """ Get a list of deletions needed across tables for deleting an entity.
  
    Args:
      entity: An entity object.
      composite_indices: A list or tuple of composite indices.
    Returns:
      A list of dictionaries representing mutation operations.
    """
    deletions = []
    app_id = clean_app_id(entity.key().app())
    namespace = entity.key().name
  
  [... the rest is omitted ...]
files.new_path: AppDB/appscale/datastore/cassandra_env/cassandra_interface.py
files.old_contents: # Programmer: Navraj Chohan <nlake44@gmail.com>
  
  """
   Cassandra Interface for AppScale
  """
  import cassandra
  import logging
  import sys
  import time
  
  from cassandra.cluster import Cluster
  from cassandra.concurrent import execute_concurrent
  from cassandra.policies import FallthroughRetryPolicy
  from cassandra.policies import RetryPolicy
  from cassandra.query import BatchStatement
  from cassandra.query import ConsistencyLevel
  from cassandra.query import SimpleStatement
  from cassandra.query import ValueSequence
  from .. import dbconstants
  from .. import helper_functions
  from ..dbconstants import AppScaleDBConnectionError
  from ..dbconstants import TxnActions
  from ..dbinterface import AppDBInterface
  from ..unpackaged import APPSCALE_LIB_DIR
  from ..utils import clean_app_id
  from ..utils import encode_index_pb
  from ..utils import get_composite_index_keys
  from ..utils import get_composite_indexes_rows
  from ..utils import get_entity_key
  from ..utils import get_entity_kind
  from ..utils import get_index_kv_from_tuple
  from ..utils import get_kind_key
  
  sys.path.append(APPSCALE_LIB_DIR)
  import appscale_info
  
  # The directory Cassandra is installed to.
  CASSANDRA_INSTALL_DIR = '/opt/cassandra'
  
  # Full path for the nodetool binary.
  NODE_TOOL = '{}/cassandra/bin/nodetool'.format(CASSANDRA_INSTALL_DIR)
  
  # The keyspace used for all tables
  KEYSPACE = "Keyspace1"
  
  # Cassandra watch name.
  CASSANDRA_MONIT_WATCH_NAME = "cassandra-9999"
  
  # The number of times to retry connecting to Cassandra.
  INITIAL_CONNECT_RETRIES = 20
  
  # The data layout version that the datastore expects.
  EXPECTED_DATA_VERSION = 1.0
  
  # The metadata key for the data layout version.
  VERSION_INFO_KEY = 'version'
  
  # The metadata key indicating that the database has been primed.
  PRIMED_KEY = 'primed'
  
  # The size in bytes that a batch must be to use the batches table.
  LARGE_BATCH_THRESHOLD = 5 << 10
  
  
  def batch_size(batch):
    """ Calculates the size of a batch.
  
    Args:
      batch: A list of dictionaries representing mutations.
    Returns:
      An integer specifying the size in bytes of the batch.
    """
    size = 0
    for mutation in batch:
      size += len(mutation['key'])
      if 'values' in mutation:
        for value in mutation['values'].values():
          size += len(value)
    return size
  
  
  def deletions_for_entity(entity, composite_indices=()):
    """ Get a list of deletions needed across tables for deleting an entity.
  
    Args:
      entity: An entity object.
      composite_indices: A list or tuple of composite indices.
    Returns:
      A list of dictionaries representing mutation operations.
    """
    deletions = []
    app_id = clean_app_id(entity.key().app())
    namespace = entity.key().name_space()
    prefix = dbconstants
  
  [... the rest is omitted ...]
files.old_path: AppDB/appscale/datastore/cassandra_env/cassandra_interface.py
files.new_contents: """
   Datastore Constants
  """
  import cassandra.cluster
  
  SECRET_LOCATION = "/etc/appscale/secret.key"
  
  ERROR_DEFAULT = "DB_ERROR:"
  NONEXISTANT_TRANSACTION = "0"
  
  # The datastore's default HTTP port.
  DEFAULT_PORT = 4080
  
  # The datastore's default HTTPS port.
  DEFAULT_SSL_PORT = 8443
  
  # The lowest character to separate different fields in a row key.
  KEY_DELIMITER = '\x00'
  
  # The character used to separate kinds in an ancestry.
  KIND_SEPARATOR = '\x01'
  
  # HTTP code to indicate that the request is invalid.
  HTTP_BAD_REQUEST = 400
  
  # The length of an ID string. A constant length allows lexicographic ordering.
  ID_KEY_LENGTH = 10
  
  # The character between the kind and the ID/name of an entity.
  ID_SEPARATOR = ":"
  
  # The maximum number of composite indexes an application can have.
  MAX_NUMBER_OF_COMPOSITE_INDEXES = 1000
  
  # A string used to create end keys when doing range queries.
  TERMINATING_STRING = chr(255) * 500
  
  # Tombstone value for soft deletes.
  TOMBSTONE = "APPSCALE_SOFT_DELETE"
  
  TRANSIENT_CASSANDRA_ERRORS = (
    cassandra.Unavailable, cassandra.Timeout, cassandra.CoordinationFailure,
    cassandra.OperationTimedOut, cassandra.cluster.NoHostAvailable)
  
  # The database backends supported by the AppScale datastore.
  VALID_DATASTORES = ['cassandra']
  
  # Table names
  USERS_TABLE = "USERS__"
  APPS_TABLE = "APPS__"
  JOURNAL_TABLE = "JOURNAL__"
  
  ASC_PROPERTY_TABLE = "ASC_PROPERTY__"
  DSC_PROPERTY_TABLE = "DSC_PROPERTY__"
  COMPOSITE_TABLE = "COMPOSITE_INDEXES__"
  APP_ID_TABLE = "APP_IDS__"
  APP_ENTITY_TABLE = "ENTITIES__"
  APP_KIND_TABLE = "KINDS__"
  METADATA_TABLE = "METADATA__"
  DATASTORE_METADATA_TABLE = "DATASTORE_METADATA__"
  TRANSACTIONS_TABLE = 'TRANSACTIONS__'
  SCHEMA_TABLE = '__key__'
  
  INITIAL_TABLES = [ASC_PROPERTY_TABLE,
                    DSC_PROPERTY_TABLE,
                    APP_ID_TABLE,
                    APP_ENTITY_TABLE,
                    APP_KIND_TABLE,
                    COMPOSITE_TABLE,
                    METADATA_TABLE,
                    USERS_TABLE,
                    APPS_TABLE,
                    SCHEMA_TABLE,
                    DATASTORE_METADATA_TABLE,
                    TRANSACTIONS_TABLE]
  
  ###########################################
  # DB schemas for version 1 of the datastore
  ###########################################
  JOURNAL_SCHEMA = [
    "Encoded_Entity"]
  
  ENTITY_TABLE_SCHEMA = [
    "Encoded_Entity",
    "Txn_Num"]
  
  ########################################
  
  [... the rest is omitted ...]
files.new_path: AppDB/appscale/datastore/dbconstants.py
files.old_contents: """
   Datastore Constants
  """
  import cassandra.cluster
  
  SECRET_LOCATION = "/etc/appscale/secret.key"
  
  ERROR_DEFAULT = "DB_ERROR:"
  NONEXISTANT_TRANSACTION = "0"
  
  # The datastore's default HTTP port.
  DEFAULT_PORT = 4080
  
  # The datastore's default HTTPS port.
  DEFAULT_SSL_PORT = 8443
  
  # The lowest character to separate different fields in a row key.
  KEY_DELIMITER = '\x00'
  
  # The character used to separate kinds in an ancestry.
  KIND_SEPARATOR = '\x01'
  
  # HTTP code to indicate that the request is invalid.
  HTTP_BAD_REQUEST = 400
  
  # The length of an ID string. A constant length allows lexicographic ordering.
  ID_KEY_LENGTH = 10
  
  # The character between the kind and the ID/name of an entity.
  ID_SEPARATOR = ":"
  
  # The maximum number of composite indexes an application can have.
  MAX_NUMBER_OF_COMPOSITE_INDEXES = 1000
  
  # A string used to create end keys when doing range queries.
  TERMINATING_STRING = chr(255) * 500
  
  # Tombstone value for soft deletes.
  TOMBSTONE = "APPSCALE_SOFT_DELETE"
  
  TRANSIENT_CASSANDRA_ERRORS = (
    cassandra.Unavailable, cassandra.Timeout, cassandra.CoordinationFailure,
    cassandra.OperationTimedOut, cassandra.cluster.NoHostAvailable)
  
  # The database backends supported by the AppScale datastore.
  VALID_DATASTORES = ['cassandra']
  
  # Table names
  USERS_TABLE = "USERS__"
  APPS_TABLE = "APPS__"
  JOURNAL_TABLE = "JOURNAL__"
  
  ASC_PROPERTY_TABLE = "ASC_PROPERTY__"
  DSC_PROPERTY_TABLE = "DSC_PROPERTY__"
  COMPOSITE_TABLE = "COMPOSITE_INDEXES__"
  APP_ID_TABLE = "APP_IDS__"
  APP_ENTITY_TABLE = "ENTITIES__"
  APP_KIND_TABLE = "KINDS__"
  METADATA_TABLE = "METADATA__"
  DATASTORE_METADATA_TABLE = "DATASTORE_METADATA__"
  TRANSACTIONS_TABLE = 'TRANSACTIONS__'
  SCHEMA_TABLE = '__key__'
  
  INITIAL_TABLES = [ASC_PROPERTY_TABLE,
                    DSC_PROPERTY_TABLE,
                    APP_ID_TABLE,
                    APP_ENTITY_TABLE,
                    APP_KIND_TABLE,
                    COMPOSITE_TABLE,
                    METADATA_TABLE,
                    USERS_TABLE,
                    APPS_TABLE,
                    SCHEMA_TABLE,
                    DATASTORE_METADATA_TABLE,
                    TRANSACTIONS_TABLE]
  
  ###########################################
  # DB schemas for version 1 of the datastore
  ###########################################
  JOURNAL_SCHEMA = [
    "Encoded_Entity"]
  
  ENTITY_TABLE_SCHEMA = [
    "Encoded_Entity",
    "Txn_Num"]
  
  ########################################
  
  [... the rest is omitted ...]
files.old_path: AppDB/appscale/datastore/dbconstants.py
files.new_contents: #!/usr/bin/env python
  # Programmer: Navraj Chohan <nlake44@gmail.com>
  
  import sys
  import unittest
  
  from appscale.datastore import dbconstants
  from appscale.datastore import utils
  from appscale.datastore.datastore_distributed import DatastoreDistributed
  from appscale.datastore.dbconstants import APP_ENTITY_SCHEMA
  from appscale.datastore.dbconstants import ID_KEY_LENGTH
  from appscale.datastore.dbconstants import JOURNAL_SCHEMA
  from appscale.datastore.dbconstants import TOMBSTONE
  from appscale.datastore.cassandra_env.cassandra_interface import DatastoreProxy
  from appscale.datastore.cassandra_env.cassandra_interface import\
    deletions_for_entity
  from appscale.datastore.cassandra_env.cassandra_interface import\
    index_deletions
  from appscale.datastore.cassandra_env.cassandra_interface import\
    mutations_for_entity
  from appscale.datastore.unpackaged import APPSCALE_LIB_DIR
  from appscale.datastore.unpackaged import APPSCALE_PYTHON_APPSERVER
  from appscale.datastore.utils import encode_index_pb
  from appscale.datastore.utils import get_entity_key
  from appscale.datastore.utils import get_entity_kind
  from appscale.datastore.utils import get_index_key_from_params
  from appscale.datastore.utils import get_index_kv_from_tuple
  from appscale.datastore.utils import get_kind_key
  from appscale.datastore.zkappscale.zktransaction import TX_TIMEOUT
  from appscale.datastore.zkappscale.zktransaction import ZKTransactionException
  from cassandra.cluster import Cluster
  from flexmock import flexmock
  
  sys.path.append(APPSCALE_PYTHON_APPSERVER)
  from google.appengine.api import api_base_pb
  from google.appengine.datastore import entity_pb
  from google.appengine.datastore import datastore_pb
  from google.appengine.ext import db
  
  sys.path.append(APPSCALE_LIB_DIR)
  import appscale_info
  
  
  class Item(db.Model):
    name = db.StringProperty(required = True)
  
  
  class TestDatastoreServer(unittest.TestCase):
    """
    A set of test cases for the datastore server (datastore server v2)
    """
    BASIC_ENTITY = ['guestbook', 'Greeting', 'foo', 'content', 'hello world']
  
    def get_zookeeper(self):
      zookeeper = flexmock()
      zookeeper.should_receive("acquire_lock").and_return(True)
      zookeeper.should_receive("release_lock").and_return(True)
      zookeeper.should_receive("get_transaction_id").and_return(1)
      zookeeper.should_receive("increment_and_get_counter").and_return(0,1000)
      return zookeeper
  
    def test_get_entity_kind(self):
      db_batch = flexmock()
      db_batch.should_receive('valid_data_version
  
  [... the rest is omitted ...]
files.new_path: AppDB/test/unit/test_datastore_server.py
files.old_contents: #!/usr/bin/env python
  # Programmer: Navraj Chohan <nlake44@gmail.com>
  
  import sys
  import unittest
  
  from appscale.datastore import dbconstants
  from appscale.datastore import utils
  from appscale.datastore.datastore_distributed import DatastoreDistributed
  from appscale.datastore.dbconstants import APP_ENTITY_SCHEMA
  from appscale.datastore.dbconstants import ID_KEY_LENGTH
  from appscale.datastore.dbconstants import JOURNAL_SCHEMA
  from appscale.datastore.dbconstants import TOMBSTONE
  from appscale.datastore.cassandra_env.cassandra_interface import DatastoreProxy
  from appscale.datastore.cassandra_env.cassandra_interface import\
    deletions_for_entity
  from appscale.datastore.cassandra_env.cassandra_interface import\
    index_deletions
  from appscale.datastore.cassandra_env.cassandra_interface import\
    mutations_for_entity
  from appscale.datastore.unpackaged import APPSCALE_LIB_DIR
  from appscale.datastore.unpackaged import APPSCALE_PYTHON_APPSERVER
  from appscale.datastore.utils import encode_index_pb
  from appscale.datastore.utils import get_entity_key
  from appscale.datastore.utils import get_entity_kind
  from appscale.datastore.utils import get_index_key_from_params
  from appscale.datastore.utils import get_index_kv_from_tuple
  from appscale.datastore.utils import get_kind_key
  from appscale.datastore.zkappscale.zktransaction import TX_TIMEOUT
  from appscale.datastore.zkappscale.zktransaction import ZKTransactionException
  from cassandra.cluster import Cluster
  from flexmock import flexmock
  
  sys.path.append(APPSCALE_PYTHON_APPSERVER)
  from google.appengine.api import api_base_pb
  from google.appengine.datastore import entity_pb
  from google.appengine.datastore import datastore_pb
  from google.appengine.ext import db
  
  sys.path.append(APPSCALE_LIB_DIR)
  import appscale_info
  
  
  class Item(db.Model):
    name = db.StringProperty(required = True)
  
  
  class TestDatastoreServer(unittest.TestCase):
    """
    A set of test cases for the datastore server (datastore server v2)
    """
    BASIC_ENTITY = ['guestbook', 'Greeting', 'foo', 'content', 'hello world']
  
    def get_zookeeper(self):
      zookeeper = flexmock()
      zookeeper.should_receive("acquire_lock").and_return(True)
      zookeeper.should_receive("release_lock").and_return(True)
      zookeeper.should_receive("get_transaction_id").and_return(1)
      zookeeper.should_receive("increment_and_get_counter").and_return(0,1000)
      return zookeeper
  
    def test_get_entity_kind(self):
      db_batch = flexmock()
      db_batch.should_receive('valid_data_version
  
  [... the rest is omitted ...]
files.old_path: AppDB/test/unit/test_datastore_server.py