files.new_contents: """
  This module contains shared encoding and decoding utilities for translating
  datastore types to bytestrings and vice versa.
  """
  import bisect
  import logging
  import struct
  import sys
  
  import six
  
  from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
  from appscale.datastore.dbconstants import BadRequest, InternalError
  
  sys.path.append(APPSCALE_PYTHON_APPSERVER)
  from google.appengine.datastore import entity_pb
  
  # The number of bytes used to encode a read version in a way that is comparable
  # to commit versionstamps.
  READ_VERSION_SIZE = 8
  
  # Byte values that signify a data type. A couple values are left between each
  # one just in case they need adjustment.
  NULL_CODE = 0x01
  # 17 values are reserved for integers in order to allow variable size in both a
  # negative and positive direction.
  INT64_ZERO_CODE = 0x0C
  FALSE_CODE = 0x17
  TRUE_CODE = 0x18
  BYTES_CODE = 0x1B
  DOUBLE_CODE = 0x1E
  POINT_CODE = 0x21
  USER_CODE = 0x24
  REFERENCE_CODE = 0x27
  
  # These are defined for clarity when selecting property type limits.
  MIN_INT64_CODE = INT64_ZERO_CODE - 8
  MAX_INT64_CODE = INT64_ZERO_CODE + 8
  
  # Ensures the shorter of two variable-length values (with identical prefixes)
  # is placed before the longer one. Otherwise, the following byte(s) could
  # determine the sort order. It also allows a decoder to find the end of the
  # value.
  TERMINATOR = 0x00
  
  logger = logging.getLogger(__name__)
  
  
  def reverse_bits(blob):
    return b''.join(map(lambda x: six.int2byte(x ^ 0xFF), six.iterbytes(blob)))
  
  
  def encode_marker(marker, reverse):
    return six.int2byte(marker ^ 0xFF) if reverse else six.int2byte(marker)
  
  
  def decode_str(string):
    """ Converts byte strings to unicode strings. """
    if isinstance(string, six.text_type):
      return string
  
    return string.decode('utf-8')
  
  
  class Int64(object):
    """
    Handles encoding and decoding operations for 64-bit integers. The encoded
    value is variable-length. The value of the marker indicates how many bytes
    were used to encode the value.
    """
    LIMITS = tuple((1 << (i * 8)) - 1 for i in range(9))
  
    @classmethod
    def encode(cls, value, reverse=False):
      if value == 0:
        code = INT64_ZERO_CODE
        packed = b''
      elif value > 0:
        encoded_size = bisect.bisect_left(cls.LIMITS, value)
        adjusted_value = cls.LIMITS[encoded_size] - value if reverse else value
        code = INT64_ZERO_CODE + encoded_size
        packed = struct.pack('>Q', adjusted_value)[-
  
  [... the rest is omitted ...]
files.new_path: AppDB/appscale/datastore/fdb/codecs.py
files.old_contents: """
  This module contains shared encoding and decoding utilities for translating
  datastore types to bytestrings and vice versa.
  """
  import bisect
  import logging
  import struct
  import sys
  
  import six
  
  from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
  from appscale.datastore.dbconstants import BadRequest, InternalError
  
  sys.path.append(APPSCALE_PYTHON_APPSERVER)
  from google.appengine.datastore import entity_pb
  
  # The number of bytes used to encode a read version in a way that is comparable
  # to commit versionstamps.
  READ_VERSION_SIZE = 8
  
  # Byte values that signify a data type. A couple values are left between each
  # one just in case they need adjustment.
  NULL_CODE = 0x01
  # 17 values are reserved for integers in order to allow variable size in both a
  # negative and positive direction.
  INT64_ZERO_CODE = 0x0C
  FALSE_CODE = 0x17
  TRUE_CODE = 0x18
  BYTES_CODE = 0x1B
  DOUBLE_CODE = 0x1E
  POINT_CODE = 0x21
  USER_CODE = 0x24
  REFERENCE_CODE = 0x27
  
  # Ensures the shorter of two variable-length values (with identical prefixes)
  # is placed before the longer one. Otherwise, the following byte(s) could
  # determine the sort order. It also allows a decoder to find the end of the
  # value.
  TERMINATOR = 0x00
  
  logger = logging.getLogger(__name__)
  
  
  def reverse_bits(blob):
    return b''.join(map(lambda x: six.int2byte(x ^ 0xFF), six.iterbytes(blob)))
  
  
  def encode_marker(marker, reverse):
    return six.int2byte(marker ^ 0xFF) if reverse else six.int2byte(marker)
  
  
  def decode_str(string):
    """ Converts byte strings to unicode strings. """
    if isinstance(string, six.text_type):
      return string
  
    return string.decode('utf-8')
  
  
  class Int64(object):
    """
    Handles encoding and decoding operations for 64-bit integers. The encoded
    value is variable-length. The value of the marker indicates how many bytes
    were used to encode the value.
    """
    LIMITS = tuple((1 << (i * 8)) - 1 for i in range(9))
  
    @classmethod
    def encode(cls, value, reverse=False):
      if value == 0:
        code = INT64_ZERO_CODE
        packed = b''
      elif value > 0:
        encoded_size = bisect.bisect_left(cls.LIMITS, value)
        adjusted_value = cls.LIMITS[encoded_size] - value if reverse else value
        code = INT64_ZERO_CODE + encoded_size
        packed = struct.pack('>Q', adjusted_value)[-encoded_size:]
      else:
        encoded_size = bisect.bisect_left(cls.LIMITS, -value)
        # Shift negative values to an unsigned space.
        adjusted_value = -value
  
  [... the rest is omitted ...]
files.old_path: AppDB/appscale/datastore/fdb/codecs.py
files.new_contents: """
  This module stores and retrieves entity data as well as the metadata needed to
  achieve snapshot isolation during transactions. The DataManager is the main
  interface that clients can use to interact with the data layer. See its
  documentation for implementation details.
  """
  from __future__ import division
  import logging
  import math
  import sys
  
  import six
  import six.moves as sm
  from tornado import gen
  
  from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
  from appscale.datastore.fdb.codecs import (
    decode_str, encode_versionstamp_index, Int64, Path, Text)
  from appscale.datastore.fdb.utils import (
    ABSENT_VERSION, DS_ROOT, fdb, hash_tuple, ResultIterator, VERSIONSTAMP_SIZE)
  
  sys.path.append(APPSCALE_PYTHON_APPSERVER)
  from google.appengine.datastore import entity_pb
  
  logger = logging.getLogger(__name__)
  
  first_gt_or_equal = fdb.KeySelector.first_greater_or_equal
  
  
  class VersionEntry(object):
    """ Encapsulates details for an entity version. """
    INCOMPLETE = 1
    __slots__ = ['project_id', 'namespace', 'path', 'commit_versionstamp',
                 'version', '_encoded_entity', '_decoded_entity']
  
    def __init__(self, project_id, namespace, path, commit_versionstamp=None,
                 version=None, encoded_entity=None):
      self.project_id = project_id
      self.namespace = namespace
      self.path = path
      self.commit_versionstamp = commit_versionstamp
      self.version = ABSENT_VERSION if version is None else version
      self._encoded_entity = encoded_entity
      self._decoded_entity = None
  
    def __repr__(self):
      # Since the encoded entity can be large, it probably does not make sense to
      # include it in the string representation.
      blob_repr = self._encoded_entity
      if blob_repr is not None:
        blob_repr = u'<bytes object with length={}>'.format(len(blob_repr))
  
      return u'VersionEntry({!r}, {!r}, {!r}, {!r}, {!r}, {})'.format(
        self.project_id, self.namespace, self.path, self.commit_versionstamp,
        self.version, blob_repr)
  
    @property
    def present(self):
      return self.commit_versionstamp is not None
  
    @property
    def complete(self):
      return self._encoded_entity != self.INCOMPLETE
  
    @property
    def has_entity(self):
      return bool(self._encoded_entity)
  
    @property
    def key(self):
      key = entity_pb.Reference()
      key.set_app(self.project_id)
      if self.namespace is not None:
        key.set_name_space(self.namespace)
  
      key.mutable_path().MergeFrom(Path.decode(self.path))
      return key
  
    @property
    def encoded(self):
      if not self.complete:
        raise ValueError(u'
  
  [... the rest is omitted ...]
files.new_path: AppDB/appscale/datastore/fdb/data.py
files.old_contents: """
  This module stores and retrieves entity data as well as the metadata needed to
  achieve snapshot isolation during transactions. The DataManager is the main
  interface that clients can use to interact with the data layer. See its
  documentation for implementation details.
  """
  from __future__ import division
  import logging
  import math
  import sys
  
  import six
  import six.moves as sm
  from tornado import gen
  
  from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
  from appscale.datastore.fdb.codecs import (
    decode_str, encode_versionstamp_index, Int64, Path, Text)
  from appscale.datastore.fdb.utils import (
    ABSENT_VERSION, DS_ROOT, fdb, hash_tuple, ResultIterator, VERSIONSTAMP_SIZE)
  
  sys.path.append(APPSCALE_PYTHON_APPSERVER)
  from google.appengine.datastore import entity_pb
  
  logger = logging.getLogger(__name__)
  
  first_gt_or_equal = fdb.KeySelector.first_greater_or_equal
  
  
  class VersionEntry(object):
    """ Encapsulates details for an entity version. """
    INCOMPLETE = 1
    __slots__ = ['project_id', 'namespace', 'path', 'commit_versionstamp',
                 'version', '_encoded_entity', '_decoded_entity']
  
    def __init__(self, project_id, namespace, path, commit_versionstamp=None,
                 version=None, encoded_entity=None):
      self.project_id = project_id
      self.namespace = namespace
      self.path = path
      self.commit_versionstamp = commit_versionstamp
      self.version = ABSENT_VERSION if version is None else version
      self._encoded_entity = encoded_entity
      self._decoded_entity = None
  
    def __repr__(self):
      # Since the encoded entity can be large, it probably does not make sense to
      # include it in the string representation.
      blob_repr = self._encoded_entity
      if blob_repr is not None:
        blob_repr = u'<bytes object with length={}>'.format(len(blob_repr))
  
      return u'VersionEntry({!r}, {!r}, {!r}, {!r}, {!r}, {})'.format(
        self.project_id, self.namespace, self.path, self.commit_versionstamp,
        self.version, blob_repr)
  
    @property
    def present(self):
      return self.commit_versionstamp is not None
  
    @property
    def complete(self):
      return self._encoded_entity != self.INCOMPLETE
  
    @property
    def has_entity(self):
      return bool(self._encoded_entity)
  
    @property
    def key(self):
      key = entity_pb.Reference()
      key.set_app(self.project_id)
      if self.namespace is not None:
        key.set_name_space(self.namespace)
  
      key.mutable_path().MergeFrom(Path.decode(self.path))
      return key
  
    @property
    def encoded(self):
      if not self.complete:
        raise ValueError(u'
  
  [... the rest is omitted ...]
files.old_path: AppDB/appscale/datastore/fdb/data.py
files.new_contents: """
  This module stores and queries entity indexes. The IndexManager is the main
  interface that clients can use to interact with the index layer. See its
  documentation for implementation details.
  """
  from __future__ import division
  
  import itertools
  import logging
  import monotonic
  import sys
  
  import six
  from tornado import gen
  
  from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
  from appscale.datastore.fdb import codecs
  from appscale.datastore.fdb.codecs import (
    decode_str, decode_value, encode_value, encode_versionstamp_index, Path)
  from appscale.datastore.fdb.sdk import FindIndexToUse, ListCursor
  from appscale.datastore.fdb.utils import (
    format_prop_val, DS_ROOT, fdb, get_scatter_val, MAX_FDB_TX_DURATION,
    ResultIterator, SCATTER_PROP, VERSIONSTAMP_SIZE)
  from appscale.datastore.dbconstants import BadRequest, InternalError
  from appscale.datastore.index_manager import IndexInaccessible
  
  sys.path.append(APPSCALE_PYTHON_APPSERVER)
  from google.appengine.datastore import datastore_pb, entity_pb
  from google.appengine.datastore.datastore_pb import Query_Filter, Query_Order
  
  logger = logging.getLogger(__name__)
  
  KEY_PROP = u'__key__'
  
  first_gt_or_equal = fdb.KeySelector.first_greater_or_equal
  
  
  class FilterProperty(object):
    """ Encapsulates details for a FilterProperty that came from a query. """
    __slots__ = ['name', 'filters']
  
    def __init__(self, prop_name, filters):
      self.name = prop_name
      self.filters = filters
  
    @property
    def equality(self):
      return all(op == Query_Filter.EQUAL for op, _ in self.filters)
  
    def __repr__(self):
      return u'FilterProperty(%r, %r)' % (self.name, self.filters)
  
  
  def group_filters(query):
    filter_props = []
    for query_filter in query.filter_list():
      if query_filter.property_size() != 1:
        raise BadRequest(u'Each filter must have exactly one property')
  
      prop = query_filter.property(0)
      prop_name = decode_str(prop.name())
      filter_info = (query_filter.op(), prop.value())
      if filter_props and filter_props[-1].name == prop_name:
        filter_props[-1].filters.append(filter_info)
      else:
        filter_props.append(FilterProperty(prop_name, [filter_info]))
  
    # Since the filter list can come in any order, put inequality filters last.
    inequality_index = None
    for index, filter_prop in enumerate(filter_props):
      if not filter_prop.equality:
        inequality_index = index
        break
  
    if inequality_index is not None:
      inequality_prop = filter_props.pop(inequality_index)
      filter_props.append(inequality_prop)
  
    # Put key filters last.
    key_index = None
    for index,
  
  [... the rest is omitted ...]
files.new_path: AppDB/appscale/datastore/fdb/indexes.py
files.old_contents: """
  This module stores and queries entity indexes. The IndexManager is the main
  interface that clients can use to interact with the index layer. See its
  documentation for implementation details.
  """
  from __future__ import division
  
  import itertools
  import logging
  import monotonic
  import sys
  
  import six
  from tornado import gen
  
  from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
  from appscale.datastore.fdb.codecs import (
    decode_str, decode_value, encode_value, encode_versionstamp_index, Path)
  from appscale.datastore.fdb.sdk import FindIndexToUse, ListCursor
  from appscale.datastore.fdb.utils import (
    format_prop_val, DS_ROOT, fdb, get_scatter_val, MAX_FDB_TX_DURATION,
    ResultIterator, SCATTER_PROP, VERSIONSTAMP_SIZE)
  from appscale.datastore.dbconstants import BadRequest, InternalError
  from appscale.datastore.index_manager import IndexInaccessible
  
  sys.path.append(APPSCALE_PYTHON_APPSERVER)
  from google.appengine.datastore import datastore_pb, entity_pb
  from google.appengine.datastore.datastore_pb import Query_Filter, Query_Order
  
  logger = logging.getLogger(__name__)
  
  KEY_PROP = u'__key__'
  
  first_gt_or_equal = fdb.KeySelector.first_greater_or_equal
  
  
  class FilterProperty(object):
    """ Encapsulates details for a FilterProperty that came from a query. """
    __slots__ = ['name', 'filters']
  
    def __init__(self, prop_name, filters):
      self.name = prop_name
      self.filters = filters
  
    @property
    def equality(self):
      return all(op == Query_Filter.EQUAL for op, _ in self.filters)
  
    def __repr__(self):
      return u'FilterProperty(%r, %r)' % (self.name, self.filters)
  
  
  def group_filters(query):
    filter_props = []
    for query_filter in query.filter_list():
      if query_filter.property_size() != 1:
        raise BadRequest(u'Each filter must have exactly one property')
  
      prop = query_filter.property(0)
      prop_name = decode_str(prop.name())
      filter_info = (query_filter.op(), prop.value())
      if filter_props and filter_props[-1].name == prop_name:
        filter_props[-1].filters.append(filter_info)
      else:
        filter_props.append(FilterProperty(prop_name, [filter_info]))
  
    # Since the filter list can come in any order, put inequality filters last.
    inequality_index = None
    for index, filter_prop in enumerate(filter_props):
      if not filter_prop.equality:
        inequality_index = index
        break
  
    if inequality_index is not None:
      inequality_prop = filter_props.pop(inequality_index)
      filter_props.append(inequality_prop)
  
    # Put key filters last.
    key_index = None
    for index, filter_prop in enumerate(filter_props):
      if filter
  
  [... the rest is omitted ...]
files.old_path: AppDB/appscale/datastore/fdb/indexes.py