files.new_contents: import datetime
  import logging
  import os
  import random
  import re
  import sys
  import threading
  import time
  
  import appscale_datastore_batch
  import dbconstants
  import utils
  
  from appscale.common import appscale_info
  from appscale.common import constants
  from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
  from appscale.common.unpackaged import DASHBOARD_DIR
  from appscale.taskqueue.distributed_tq import TaskName
  from .cassandra_env import cassandra_interface
  from .datastore_distributed import DatastoreDistributed
  from .utils import get_composite_indexes_rows
  from .zkappscale import zktransaction as zk
  from .zkappscale.entity_lock import EntityLock
  
  sys.path.append(APPSCALE_PYTHON_APPSERVER)
  from google.appengine.api import apiproxy_stub_map
  from google.appengine.api import datastore_distributed
  from google.appengine.api.memcache import memcache_distributed
  from google.appengine.datastore import datastore_pb
  from google.appengine.datastore import entity_pb
  from google.appengine.datastore.datastore_query import Cursor
  from google.appengine.ext import db
  from google.appengine.ext.db import stats
  from google.appengine.ext.db import metadata
  from google.appengine.api import datastore_errors
  
  sys.path.append(os.path.join(DASHBOARD_DIR, 'lib'))
  from dashboard_logs import RequestLogLine
  
  
  class DatastoreGroomer(threading.Thread):
    """ Scans the entire database for each application. """
  
    # The amount of seconds between polling to get the groomer lock.
    # Each datastore server does this poll, so it happens the number
    # of datastore servers within this lock period.
    LOCK_POLL_PERIOD = 4 * 60 * 60 # <- 4 hours
  
    # Retry sleep on datastore error in seconds.
    DB_ERROR_PERIOD = 30
  
    # The number of entities retrieved in a datastore request.
    BATCH_SIZE = 100
  
    # Any kind that is of __*__ is private and should not have stats.
    PRIVATE_KINDS = '__(.*)__'
  
    # Any kind that is of _*_ is protected and should not have stats.
    PROTECTED_KINDS = '_(.*)_'
  
    # The amount of time in seconds before we want to clean up task name holders.
    TASK_NAME_TIMEOUT = 24 * 60 * 60
  
    # The amount of time before logs are considered too old.
    LOG_STORAGE_TIMEOUT = 24 * 60 * 60 * 7
  
    # Do not generate stats for AppScale internal apps.
    APPSCALE_APPLICATIONS = ['apichecker', 'appscaledashboard']
  
    # A sentinel value to signify that this app does not have composite indexes.
    NO_COMPOSITES = "NO_COMPS_INDEXES_HERE"
  
    # The path in ZooKeeper where the groomer state is stored.
    GROOMER_STATE_PATH = '/appscale/groomer_state'
  
    # The characters used to separate values when storing the groomer state.
    GROOMER_STATE_DELIMITER = '||'
  
    # The ID for the task to clean up entities.
    CLEAN_ENTITIES_TASK = 'entities'
  
    # The ID for the task to clean up ascending indices.
    CLEAN_ASC_INDICES_TASK = 'asc-indices'
  
    # The ID for the task to clean up descending indices.
    CLEAN_DSC_INDICES_TASK = 'dsc-indices'
  
    # The ID for the task to clean up kind indices.
    CLEAN_KIND_INDICES_TASK = 'kind-indices'
  
    # The ID for the task to clean up old logs.
    CLEAN_LOGS_TASK = 'logs'
  
    # The ID for the task to clean up old tasks.
    CLEAN_TASKS_TASK = 'tasks'
  
    # Log progress every time this many seconds have passed.
    LOG_PROGRESS_FREQUENCY = 60 * 5
  
    def __init__(self, zoo_keeper, table_name, ds_path):
      """ Constructor.
  
      Args:
        zk: ZooKeeper client.
        table_name: The database used (ie, cassandra)
        ds_path: The connection path to the datastore_server.
      """
      logging.info("Logging started")
  
      threading.Thread.__init__(self)
      self.zoo_keeper = zoo_keeper
      self.table_name = table_name
      self.db_access = None
      self.ds_access = None
      self.datastore_path = ds_path
      self.stats = {}
      self.namespace_info = {}
      self.num_deletes = 0
      self.composite_index_cache = {}
      self.entities_checked = 0
      self.journal_entries_cleaned = 0
      self.index_entries_checked = 0
      self.index_entries_delete_failures = 0
      self.index_entries_cleaned = 0
      self.last_logged = time.time()
      self.groomer_state = []
  
    def stop(self):
      """ Stops the groomer thread. """
      self.zoo_keeper.close()
  
    def run(self):
      """ Starts the main loop of the groomer thread. """
      while True:
  
        logging.debug("Trying to get groomer lock.")
        if self.get_groomer_lock():
          logging.info("Got the groomer lock.")
          self.run_groomer()
          try:
            self.zoo_keeper.release_lock_with_path(zk.DS_GROOM_LOCK_PATH)
          except zk.ZKTransactionException, zk_exception:
            logging.error("Unable to release zk lock {0}.".\
              format(str(zk_exception)))
          except zk.ZKInternalException, zk_exception:
            logging.error("Unable to release zk lock {0}.".\
              format(str(zk_exception)))
        else:
          logging.info("Did not get the groomer lock.")
        sleep_time = random.randint(1, self.LOCK_POLL_PERIOD)
        logging.info('Sleeping for {:.1f} minutes.'.format(sleep_time/60.0))
        time.sleep(sleep_time)
  
    def get_groomer_lock(self):
      """ Tries to acquire the lock to the datastore groomer.
  
      Returns:
        True on success, False otherwise.
      """
      return self.zoo_keeper.get_lock_with_path(zk.DS_GROOM_LOCK_PATH)
  
    def get_entity_batch(self, last_key):
      """ Gets a batch of entites to operate on.
  
      Args:
        last_key: The last key from a previous query.
      Returns:
        A list of entities.
      """
      return self.db_access.range_query(dbconstants.APP_ENTITY_TABLE,
        dbconstants.APP_ENTITY_SCHEMA, last_key, "", self.BATCH_SIZE,
        start_inclusive=False)
  
    def reset_statistics(self):
      """ Reinitializes statistics. """
      self.stats = {}
      self.namespace_info = {}
      self.num_deletes = 0
      self.journal_entries_cleaned = 0
  
    def hard_delete_row(self, row_key):
      """ Does a hard delete on a given row key to the entity
          table.
  
      Args:
        row_key: A str representing the row key to delete.
      Returns:
        True on success, False otherwise.
      """
      try:
        self.db_access.batch_delete(dbconstants.APP_ENTITY_TABLE,
          [row_key])
      except dbconstants.AppScaleDBConnectionError, db_error:
        logging.error("Error hard deleting key {0}-->{1}".format(
          row_key, db_error))
        return False
      except Exception, exception:
        logging.error("Caught unexcepted exception {0}".format(exception))
        return False
  
      return True
  
    def load_composite_cache(self, app_id):
      """ Load the composite index cache for an application ID.
  
      Args:
        app_id: A str, the application ID.
      Returns:
        True if the application has composites. False otherwise.
      """
      start_key = dbconstants.KEY_DELIMITER.join([app_id, 'index', ''])
      end_key = dbconstants.KEY_DELIMITER.join(
        [app_id, 'index', dbconstants.TERMINATING_STRING])
  
      results = self.db_access.range_query(dbconstants.METADATA_TABLE,
        dbconstants.METADATA_TABLE, start_key, end_key,
        dbconstants.MAX_NUMBER_OF_COMPOSITE_INDEXES)
      list_result = []
      for list_item in results:
        for _, value in list_item.iteritems():
          list_result.append(value['data'])
  
      self.composite_index_cache[app_id] = self.NO_COMPOSITES
      kind_index_dictionary = {}
      for index in list_result:
        new_index = entity_pb.CompositeIndex()
        new_index.ParseFromString(index)
        kind = new_index.definition().entity_type()
        if kind in kind_index_dictionary:
          kind_index_d
  
  [... the rest is omitted ...]
files.new_path: AppDB/appscale/datastore/groomer.py
files.old_contents: import datetime
  import logging
  import os
  import random
  import re
  import sys
  import threading
  import time
  
  import appscale_datastore_batch
  import dbconstants
  import utils
  
  from appscale.common import appscale_info
  from appscale.common import constants
  from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
  from appscale.common.unpackaged import DASHBOARD_DIR
  from appscale.taskqueue.distributed_tq import TaskName
  from .cassandra_env import cassandra_interface
  from .datastore_distributed import DatastoreDistributed
  from .utils import get_composite_indexes_rows
  from .zkappscale import zktransaction as zk
  
  sys.path.append(APPSCALE_PYTHON_APPSERVER)
  from google.appengine.api import apiproxy_stub_map
  from google.appengine.api import datastore_distributed
  from google.appengine.api.memcache import memcache_distributed
  from google.appengine.datastore import datastore_pb
  from google.appengine.datastore import entity_pb
  from google.appengine.datastore.datastore_query import Cursor
  from google.appengine.ext import db
  from google.appengine.ext.db import stats
  from google.appengine.ext.db import metadata
  from google.appengine.api import datastore_errors
  
  sys.path.append(os.path.join(DASHBOARD_DIR, 'lib'))
  from dashboard_logs import RequestLogLine
  
  
  class DatastoreGroomer(threading.Thread):
    """ Scans the entire database for each application. """
  
    # The amount of seconds between polling to get the groomer lock.
    # Each datastore server does this poll, so it happens the number
    # of datastore servers within this lock period.
    LOCK_POLL_PERIOD = 4 * 60 * 60 # <- 4 hours
  
    # Retry sleep on datastore error in seconds.
    DB_ERROR_PERIOD = 30
  
    # The number of entities retrieved in a datastore request.
    BATCH_SIZE = 100
  
    # Any kind that is of __*__ is private and should not have stats.
    PRIVATE_KINDS = '__(.*)__'
  
    # Any kind that is of _*_ is protected and should not have stats.
    PROTECTED_KINDS = '_(.*)_'
  
    # The amount of time in seconds before we want to clean up task name holders.
    TASK_NAME_TIMEOUT = 24 * 60 * 60
  
    # The amount of time before logs are considered too old.
    LOG_STORAGE_TIMEOUT = 24 * 60 * 60 * 7
  
    # Do not generate stats for AppScale internal apps.
    APPSCALE_APPLICATIONS = ['apichecker', 'appscaledashboard']
  
    # A sentinel value to signify that this app does not have composite indexes.
    NO_COMPOSITES = "NO_COMPS_INDEXES_HERE"
  
    # The path in ZooKeeper where the groomer state is stored.
    GROOMER_STATE_PATH = '/appscale/groomer_state'
  
    # The characters used to separate values when storing the groomer state.
    GROOMER_STATE_DELIMITER = '||'
  
    # The ID for the task to clean up entities.
    CLEAN_ENTITIES_TASK = 'entities'
  
    # The ID for the task to clean up ascending indices.
    CLEAN_ASC_INDICES_TASK = 'asc-indices'
  
    # The ID for the task to clean up descending indices.
    CLEAN_DSC_INDICES_TASK = 'dsc-indices'
  
    # The ID for the task to clean up kind indices.
    CLEAN_KIND_INDICES_TASK = 'kind-indices'
  
    # The ID for the task to clean up old logs.
    CLEAN_LOGS_TASK = 'logs'
  
    # The ID for the task to clean up old tasks.
    CLEAN_TASKS_TASK = 'tasks'
  
    # Log progress every time this many seconds have passed.
    LOG_PROGRESS_FREQUENCY = 60 * 5
  
    def __init__(self, zoo_keeper, table_name, ds_path):
      """ Constructor.
  
      Args:
        zk: ZooKeeper client.
        table_name: The database used (ie, cassandra)
        ds_path: The connection path to the datastore_server.
      """
      logging.info("Logging started")
  
      threading.Thread.__init__(self)
      self.zoo_keeper = zoo_keeper
      self.table_name = table_name
      self.db_access = None
      self.ds_access = None
      self.datastore_path = ds_path
      self.stats = {}
      self.namespace_info = {}
      self.num_deletes = 0
      self.composite_index_cache = {}
      self.entities_checked = 0
      self.journal_entries_cleaned = 0
      self.index_entries_checked = 0
      self.index_entries_delete_failures = 0
      self.index_entries_cleaned = 0
      self.last_logged = time.time()
      self.groomer_state = []
  
    def stop(self):
      """ Stops the groomer thread. """
      self.zoo_keeper.close()
  
    def run(self):
      """ Starts the main loop of the groomer thread. """
      while True:
  
        logging.debug("Trying to get groomer lock.")
        if self.get_groomer_lock():
          logging.info("Got the groomer lock.")
          self.run_groomer()
          try:
            self.zoo_keeper.release_lock_with_path(zk.DS_GROOM_LOCK_PATH)
          except zk.ZKTransactionException, zk_exception:
            logging.error("Unable to release zk lock {0}.".\
              format(str(zk_exception)))
          except zk.ZKInternalException, zk_exception:
            logging.error("Unable to release zk lock {0}.".\
              format(str(zk_exception)))
        else:
          logging.info("Did not get the groomer lock.")
        sleep_time = random.randint(1, self.LOCK_POLL_PERIOD)
        logging.info('Sleeping for {:.1f} minutes.'.format(sleep_time/60.0))
        time.sleep(sleep_time)
  
    def get_groomer_lock(self):
      """ Tries to acquire the lock to the datastore groomer.
  
      Returns:
        True on success, False otherwise.
      """
      return self.zoo_keeper.get_lock_with_path(zk.DS_GROOM_LOCK_PATH)
  
    def get_entity_batch(self, last_key):
      """ Gets a batch of entites to operate on.
  
      Args:
        last_key: The last key from a previous query.
      Returns:
        A list of entities.
      """
      return self.db_access.range_query(dbconstants.APP_ENTITY_TABLE,
        dbconstants.APP_ENTITY_SCHEMA, last_key, "", self.BATCH_SIZE,
        start_inclusive=False)
  
    def reset_statistics(self):
      """ Reinitializes statistics. """
      self.stats = {}
      self.namespace_info = {}
      self.num_deletes = 0
      self.journal_entries_cleaned = 0
  
    def hard_delete_row(self, row_key):
      """ Does a hard delete on a given row key to the entity
          table.
  
      Args:
        row_key: A str representing the row key to delete.
      Returns:
        True on success, False otherwise.
      """
      try:
        self.db_access.batch_delete(dbconstants.APP_ENTITY_TABLE,
          [row_key])
      except dbconstants.AppScaleDBConnectionError, db_error:
        logging.error("Error hard deleting key {0}-->{1}".format(
          row_key, db_error))
        return False
      except Exception, exception:
        logging.error("Caught unexcepted exception {0}".format(exception))
        return False
  
      return True
  
    def load_composite_cache(self, app_id):
      """ Load the composite index cache for an application ID.
  
      Args:
        app_id: A str, the application ID.
      Returns:
        True if the application has composites. False otherwise.
      """
      start_key = dbconstants.KEY_DELIMITER.join([app_id, 'index', ''])
      end_key = dbconstants.KEY_DELIMITER.join(
        [app_id, 'index', dbconstants.TERMINATING_STRING])
  
      results = self.db_access.range_query(dbconstants.METADATA_TABLE,
        dbconstants.METADATA_TABLE, start_key, end_key,
        dbconstants.MAX_NUMBER_OF_COMPOSITE_INDEXES)
      list_result = []
      for list_item in results:
        for _, value in list_item.iteritems():
          list_result.append(value['data'])
  
      self.composite_index_cache[app_id] = self.NO_COMPOSITES
      kind_index_dictionary = {}
      for index in list_result:
        new_index = entity_pb.CompositeIndex()
        new_index.ParseFromString(index)
        kind = new_index.definition().entity_type()
        if kind in kind_index_dictionary:
          kind_index_dictionary[kind].append(new_index)
        else:
  
  
  [... the rest is omitted ...]
files.old_path: AppDB/appscale/datastore/groomer.py