files.new_contents: import sys
  import uuid
  
  from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
  from .retry_policies import NO_RETRIES
  from ..dbconstants import (
    AppScaleBadArg,
    AppScaleDBConnectionError,
    TRANSIENT_CASSANDRA_ERRORS
  )
  from ..utils import logger
  from cassandra.query import (
    ConsistencyLevel,
    SimpleStatement
  )
  
  sys.path.append(APPSCALE_PYTHON_APPSERVER)
  from google.appengine.datastore.datastore_stub_util import (
    _MAX_SCATTERED_COUNTER,
    _MAX_SEQUENTIAL_COUNTER,
    ToScatteredId
  )
  
  # The number of scattered IDs the datastore should reserve at a time.
  DEFAULT_RESERVATION_SIZE = 10000
  
  
  class ReservationFailed(Exception):
    """ Indicates that a block of IDs could not be reserved. """
    pass
  
  
  class EntityIDAllocator(object):
    """ Keeps track of reserved entity IDs for a project. """
  
    def __init__(self, session, project, scattered=False):
      """ Creates a new EntityIDAllocator object.
  
      Args:
        session: A cassandra-drivers session object.
        project: A string specifying a project ID.
      """
      self.project = project
      self.session = session
      self.scattered = scattered
      if scattered:
        self.max_allowed = _MAX_SCATTERED_COUNTER
      else:
        self.max_allowed = _MAX_SEQUENTIAL_COUNTER
  
      # Allows the allocator to avoid making unnecessary Cassandra requests when
      # setting the minimum counter value.
      self._last_reserved_cache = None
  
    def _ensure_entry(self, retries=5):
      """ Ensures an entry exists for a reservation.
  
      Args:
        retries: The number of times to retry the insert.
      Raises:
        AllocationFailed if the insert is tried too many times.
      """
      if retries < 0:
        raise AppScaleDBConnectionError('Unable to create reserved_ids entry')
  
      logger.debug('Creating reserved_ids entry for {}'.format(self.project))
      insert = SimpleStatement("""
        INSERT INTO reserved_ids (project, scattered, last_reserved, op_id)
        VALUES (%(project)s, %(scattered)s, 0, uuid())
        IF NOT EXISTS
      """, retry_policy=NO_RETRIES)
      parameters = {'project': self.project, 'scattered': self.scattered}
      try:
        self.session.execute(insert, parameters)
      except TRANSIENT_CASSANDRA_ERRORS:
        return self._ensure_entry(retries=retries - 1)
  
    def _get_last_reserved(self):
      """ Retrieves the last entity ID that was reserved.
  
      Returns:
        An integer specifying an entity ID.
      """
      get_reserved = SimpleStatement("""
        SELECT last_reserved
        FROM reserved_ids
        WHERE project = %(project)s
        AND scattered = %(scattered)s
      """, consistency_level=ConsistencyLevel.SERIAL)
     
  
  [... the rest is omitted ...]
files.new_path: AppDB/appscale/datastore/cassandra_env/entity_id_allocator.py
files.old_contents: import sys
  import uuid
  
  from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
  from .retry_policies import NO_RETRIES
  from ..dbconstants import (
    AppScaleBadArg,
    AppScaleDBConnectionError,
    TRANSIENT_CASSANDRA_ERRORS
  )
  from ..utils import logger
  from cassandra.query import (
    ConsistencyLevel,
    SimpleStatement
  )
  
  sys.path.append(APPSCALE_PYTHON_APPSERVER)
  from google.appengine.datastore.datastore_stub_util import (
    _MAX_SCATTERED_COUNTER,
    _MAX_SEQUENTIAL_COUNTER,
    ToScatteredId
  )
  
  # The number of scattered IDs the datastore should reserve at a time.
  DEFAULT_RESERVATION_SIZE = 10000
  
  
  class ReservationFailed(Exception):
    """ Indicates that a block of IDs could not be reserved. """
    pass
  
  
  class EntityIDAllocator(object):
    """ Keeps track of reserved entity IDs for a project. """
  
    def __init__(self, session, project, scattered=False):
      """ Creates a new EntityIDAllocator object.
  
      Args:
        session: A cassandra-drivers session object.
        project: A string specifying a project ID.
      """
      self.project = project
      self.session = session
      self.scattered = scattered
      if scattered:
        self.max_allowed = _MAX_SCATTERED_COUNTER
      else:
        self.max_allowed = _MAX_SEQUENTIAL_COUNTER
  
    def _ensure_entry(self, retries=5):
      """ Ensures an entry exists for a reservation.
  
      Args:
        retries: The number of times to retry the insert.
      Raises:
        AllocationFailed if the insert is tried too many times.
      """
      if retries < 0:
        raise AppScaleDBConnectionError('Unable to create reserved_ids entry')
  
      logger.debug('Creating reserved_ids entry for {}'.format(self.project))
      insert = SimpleStatement("""
        INSERT INTO reserved_ids (project, scattered, last_reserved, op_id)
        VALUES (%(project)s, %(scattered)s, 0, uuid())
        IF NOT EXISTS
      """, retry_policy=NO_RETRIES)
      parameters = {'project': self.project, 'scattered': self.scattered}
      try:
        self.session.execute(insert, parameters)
      except TRANSIENT_CASSANDRA_ERRORS:
        return self._ensure_entry(retries=retries - 1)
  
    def _get_last_reserved(self):
      """ Retrieves the last entity ID that was reserved.
  
      Returns:
        An integer specifying an entity ID.
      """
      get_reserved = SimpleStatement("""
        SELECT last_reserved
        FROM reserved_ids
        WHERE project = %(project)s
        AND scattered = %(scattered)s
      """, consistency_level=ConsistencyLevel.SERIAL)
      parameters = {'project': self.project, 'scattered': self.scattered}
      try:
        result = self.session.execute(get_reserved, parameters)[0
  
  [... the rest is omitted ...]
files.old_path: AppDB/appscale/datastore/cassandra_env/entity_id_allocator.py
files.new_contents: import array
  import datetime
  import itertools
  import logging
  import md5
  import random
  import sys
  import threading
  import time
  
  import dbconstants
  import helper_functions
  
  from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
  from .dbconstants import APP_ENTITY_SCHEMA
  from .dbconstants import ID_KEY_LENGTH
  from .dbconstants import MAX_TX_DURATION
  from .cassandra_env import cassandra_interface
  from .cassandra_env.entity_id_allocator import EntityIDAllocator
  from .cassandra_env.entity_id_allocator import ScatteredAllocator
  from .utils import clean_app_id
  from .utils import encode_entity_table_key
  from .utils import encode_index_pb
  from .utils import get_composite_index_keys
  from .utils import get_entity_key
  from .utils import get_entity_kind
  from .utils import get_index_key_from_params
  from .utils import get_kind_key
  from .utils import group_for_key
  from .utils import reference_property_to_reference
  from .utils import UnprocessedQueryCursor
  from .zkappscale import entity_lock
  from .zkappscale import zktransaction
  
  sys.path.append(APPSCALE_PYTHON_APPSERVER)
  from google.appengine.api import api_base_pb
  from google.appengine.api import datastore_errors
  from google.appengine.api.datastore_distributed import _MAX_ACTIONS_PER_TXN
  from google.appengine.api.taskqueue import taskqueue_service_pb
  from google.appengine.api.taskqueue.taskqueue_distributed import\
    TaskQueueServiceStub
  from google.appengine.datastore import appscale_stub_util
  from google.appengine.datastore import datastore_pb
  from google.appengine.datastore import datastore_index
  from google.appengine.datastore import entity_pb
  from google.appengine.datastore import sortable_pb_encoder
  from google.appengine.datastore.datastore_stub_util import IdToCounter
  from google.appengine.datastore.datastore_stub_util import SEQUENTIAL
  from google.appengine.runtime import apiproxy_errors
  from google.appengine.ext import db
  from google.appengine.ext.db.metadata import Namespace
  from google.net.proto.ProtocolBuffer import ProtocolBufferDecodeError
  
  
  class DatastoreDistributed():
    """ AppScale persistent layer for the datastore API. It is the 
        replacement for the AppServers to persist their data into 
        a distributed datastore instead of a flat file.
    """
    # Max number of results for a query
    _MAXIMUM_RESULTS = 10000
  
    # The number of entries looked at when doing a composite query
    # It will keep looking at this size window when getting the result
    _MAX_COMPOSITE_WINDOW = 10000
  
    # Maximum amount of filter and orderings allowed within a query
    _MAX_QUERY_COMPONENTS = 63
  
    # For enabling and disabling range inclusivity
    _ENABLE
  
  [... the rest is omitted ...]
files.new_path: AppDB/appscale/datastore/datastore_distributed.py
files.old_contents: import array
  import datetime
  import itertools
  import logging
  import md5
  import random
  import sys
  import threading
  import time
  
  import dbconstants
  import helper_functions
  
  from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
  from .dbconstants import APP_ENTITY_SCHEMA
  from .dbconstants import ID_KEY_LENGTH
  from .dbconstants import MAX_TX_DURATION
  from .cassandra_env import cassandra_interface
  from .cassandra_env.entity_id_allocator import EntityIDAllocator
  from .cassandra_env.entity_id_allocator import ScatteredAllocator
  from .utils import clean_app_id
  from .utils import encode_entity_table_key
  from .utils import encode_index_pb
  from .utils import get_composite_index_keys
  from .utils import get_entity_key
  from .utils import get_entity_kind
  from .utils import get_index_key_from_params
  from .utils import get_kind_key
  from .utils import group_for_key
  from .utils import reference_property_to_reference
  from .utils import UnprocessedQueryCursor
  from .zkappscale import entity_lock
  from .zkappscale import zktransaction
  
  sys.path.append(APPSCALE_PYTHON_APPSERVER)
  from google.appengine.api import api_base_pb
  from google.appengine.api import datastore_errors
  from google.appengine.api.datastore_distributed import _MAX_ACTIONS_PER_TXN
  from google.appengine.api.taskqueue import taskqueue_service_pb
  from google.appengine.api.taskqueue.taskqueue_distributed import\
    TaskQueueServiceStub
  from google.appengine.datastore import appscale_stub_util
  from google.appengine.datastore import datastore_pb
  from google.appengine.datastore import datastore_index
  from google.appengine.datastore import entity_pb
  from google.appengine.datastore import sortable_pb_encoder
  from google.appengine.runtime import apiproxy_errors
  from google.appengine.ext import db
  from google.appengine.ext.db.metadata import Namespace
  from google.net.proto.ProtocolBuffer import ProtocolBufferDecodeError
  
  
  class DatastoreDistributed():
    """ AppScale persistent layer for the datastore API. It is the 
        replacement for the AppServers to persist their data into 
        a distributed datastore instead of a flat file.
    """
    # Max number of results for a query
    _MAXIMUM_RESULTS = 10000
  
    # The number of entries looked at when doing a composite query
    # It will keep looking at this size window when getting the result
    _MAX_COMPOSITE_WINDOW = 10000
  
    # Maximum amount of filter and orderings allowed within a query
    _MAX_QUERY_COMPONENTS = 63
  
    # For enabling and disabling range inclusivity
    _ENABLE_INCLUSIVITY = True
    _DISABLE_INCLUSIVITY = False
  
    # Delimiter between app names and namespace and the rest of an entity key
    _
  
  [... the rest is omitted ...]
files.old_path: AppDB/appscale/datastore/datastore_distributed.py
files.new_contents: # See LICENSE file
  #
  """
  This web service interfaces with the datastore. It takes protocol buffer
  requests from AppServers and responds according to the type of request its
  given (Put, Get, Delete, Query, etc).
  """
  import argparse
  import json
  import logging
  import os
  import sys
  import threading
  import time
  import tornado.httpserver
  import tornado.ioloop
  import tornado.web
  
  from appscale.common import appscale_info
  from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
  from .. import dbconstants
  from ..appscale_datastore_batch import DatastoreFactory
  from ..datastore_distributed import DatastoreDistributed
  from ..utils import (clean_app_id,
                       logger,
                       UnprocessedQueryResult)
  from ..zkappscale import zktransaction
  
  sys.path.append(APPSCALE_PYTHON_APPSERVER)
  from google.appengine.api import api_base_pb
  from google.appengine.api.taskqueue import taskqueue_service_pb
  from google.appengine.datastore import datastore_pb
  from google.appengine.datastore import datastore_v4_pb
  from google.appengine.datastore import entity_pb
  from google.appengine.ext.remote_api import remote_api_pb
  
  # Global for accessing the datastore. An instance of DatastoreDistributed.
  datastore_access = None
  
  # ZooKeeper global variable for locking
  zookeeper = None
  
  # Determines whether or not to allow datastore writes. Note: After enabling,
  # datastore processes must be restarted and the groomer must be stopped.
  READ_ONLY = False
  
  # Global stats.
  STATS = {}
  
  
  class ClearHandler(tornado.web.RequestHandler):
    """ Defines what to do when the webserver receives a /clear HTTP request. """
    def set_default_headers(self):
      """ Instructs clients to close the connection after each response. """
      self.set_header('Connection', 'close')
  
    @tornado.web.asynchronous
    def post(self):
      """ Handles POST requests for clearing datastore server stats. """
      global STATS
      STATS = {}
      self.write({"message": "Statistics for this server cleared."})
      self.finish()
  
  
  class ReadOnlyHandler(tornado.web.RequestHandler):
    """ Handles requests to check or set read-only mode. """
    def set_default_headers(self):
      """ Instructs clients to close the connection after each response. """
      self.set_header('Connection', 'close')
  
    @tornado.web.asynchronous
    def post(self):
      """ Handle requests to turn read-only mode on or off. """
      global READ_ONLY
  
      payload = self.request.body
      data = json.loads(payload)
      if 'readOnly' not in data:
        self.set_status(dbconstants.HTTP_BAD_REQUEST)
  
      if data['readOnly']:
        READ_ONLY = True
        message = 'Write operations now disabled.'
      else:
        READ_ONLY = False
        message = 'Write operations now enabled.'
  
      logger.info(message)
     
  
  [... the rest is omitted ...]
files.new_path: AppDB/appscale/datastore/scripts/datastore.py
files.old_contents: # See LICENSE file
  #
  """
  This web service interfaces with the datastore. It takes protocol buffer
  requests from AppServers and responds according to the type of request its
  given (Put, Get, Delete, Query, etc).
  """
  import argparse
  import json
  import logging
  import os
  import sys
  import threading
  import time
  import tornado.httpserver
  import tornado.ioloop
  import tornado.web
  
  from appscale.common import appscale_info
  from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
  from .. import dbconstants
  from ..appscale_datastore_batch import DatastoreFactory
  from ..datastore_distributed import DatastoreDistributed
  from ..utils import (clean_app_id,
                       logger,
                       UnprocessedQueryResult)
  from ..zkappscale import zktransaction
  
  sys.path.append(APPSCALE_PYTHON_APPSERVER)
  from google.appengine.api import api_base_pb
  from google.appengine.api.taskqueue import taskqueue_service_pb
  from google.appengine.datastore import datastore_pb
  from google.appengine.datastore import entity_pb
  from google.appengine.ext.remote_api import remote_api_pb
  
  # Global for accessing the datastore. An instance of DatastoreDistributed.
  datastore_access = None
  
  # ZooKeeper global variable for locking
  zookeeper = None
  
  # Determines whether or not to allow datastore writes. Note: After enabling,
  # datastore processes must be restarted and the groomer must be stopped.
  READ_ONLY = False
  
  # Global stats.
  STATS = {}
  
  
  class ClearHandler(tornado.web.RequestHandler):
    """ Defines what to do when the webserver receives a /clear HTTP request. """
    def set_default_headers(self):
      """ Instructs clients to close the connection after each response. """
      self.set_header('Connection', 'close')
  
    @tornado.web.asynchronous
    def post(self):
      """ Handles POST requests for clearing datastore server stats. """
      global STATS
      STATS = {}
      self.write({"message": "Statistics for this server cleared."})
      self.finish()
  
  
  class ReadOnlyHandler(tornado.web.RequestHandler):
    """ Handles requests to check or set read-only mode. """
    def set_default_headers(self):
      """ Instructs clients to close the connection after each response. """
      self.set_header('Connection', 'close')
  
    @tornado.web.asynchronous
    def post(self):
      """ Handle requests to turn read-only mode on or off. """
      global READ_ONLY
  
      payload = self.request.body
      data = json.loads(payload)
      if 'readOnly' not in data:
        self.set_status(dbconstants.HTTP_BAD_REQUEST)
  
      if data['readOnly']:
        READ_ONLY = True
        message = 'Write operations now disabled.'
      else:
        READ_ONLY = False
        message = 'Write operations now enabled.'
  
      logger.info(message)
      self.write({'message': message})
      self.finish()
  
  
  class MainHandler
  
  [... the rest is omitted ...]
files.old_path: AppDB/appscale/datastore/scripts/datastore.py