files.new_contents: """
  Blobstore server for uploading blobs.
  See LICENSE file.
  
  Some code was taken from 
  http://blog.doughellmann.com/2009/07/pymotw-urllib2-library-for-opening-urls.html
  
  """
  import argparse
  import base64
  import cStringIO
  import datetime
  import gzip
  import hashlib
  import itertools
  import logging
  import math
  import mimetools
  import os 
  import os.path
  import requests
  import sys
  import tornado.httpserver
  import tornado.ioloop
  import tornado.web
  import urllib
  import urllib2
  
  from appscale.common import appscale_info
  from appscale.common.constants import LOG_FORMAT
  from appscale.common.deployment_config import DeploymentConfig
  from appscale.common.deployment_config import ConfigInaccessible
  from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
  from kazoo.client import KazooClient
  from StringIO import StringIO
  
  sys.path.append(APPSCALE_PYTHON_APPSERVER)
  from google.appengine.api import apiproxy_stub_map
  from google.appengine.api import datastore_errors
  from google.appengine.api import datastore_distributed
  from google.appengine.api import datastore
  from google.appengine.api.blobstore import blobstore
  from google.appengine.api.blobstore import datastore_blob_storage
  from google.appengine.tools import dev_appserver_upload
  
  # The URL path used for uploading blobs
  UPLOAD_URL_PATH = '_ah/upload/'
  
  # The port this service binds to
  DEFAULT_PORT = 6107
  
  # The datastore kind used for storing chunks of a blob
  _BLOB_CHUNK_KIND_ = "__BlobChunk__"
  
  # Headers we ignore
  STRIPPED_HEADERS = frozenset(('content-length',
                                'content-md5',
                                'content-type',
                               ))
  
  UPLOAD_ERROR = 'There was an error with your upload. Redirect path not '\
    'found. Please contact the app owner if this persists.'
  
  # The maximum size of an incoming request.
  MAX_REQUEST_BUFF_SIZE = 2 * 1024 * 1024 * 1024  # 2GBs
  
  # The header used by GCS to provide a resumable upload ID.
  
  
  [... the rest is omitted ...]
files.new_path: AppDB/appscale/datastore/scripts/blobstore.py
files.old_contents: """
  Blobstore server for uploading blobs.
  See LICENSE file.
  
  Some code was taken from 
  http://blog.doughellmann.com/2009/07/pymotw-urllib2-library-for-opening-urls.html
  
  """
  import argparse
  import base64
  import cStringIO
  import datetime
  import gzip
  import hashlib
  import itertools
  import logging
  import math
  import mimetools
  import os 
  import os.path
  import requests
  import sys
  import tornado.httpserver
  import tornado.ioloop
  import tornado.web
  import urllib
  import urllib2
  
  from appscale.common import appscale_info
  from appscale.common.constants import LOG_FORMAT
  from appscale.common.deployment_config import DeploymentConfig
  from appscale.common.deployment_config import ConfigInaccessible
  from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
  from StringIO import StringIO
  
  sys.path.append(APPSCALE_PYTHON_APPSERVER)
  from google.appengine.api import apiproxy_stub_map
  from google.appengine.api import datastore_errors
  from google.appengine.api import datastore_distributed
  from google.appengine.api import datastore
  from google.appengine.api.blobstore import blobstore
  from google.appengine.api.blobstore import datastore_blob_storage
  from google.appengine.tools import dev_appserver_upload
  
  # The URL path used for uploading blobs
  UPLOAD_URL_PATH = '_ah/upload/'
  
  # The port this service binds to
  DEFAULT_PORT = 6107
  
  # The datastore kind used for storing chunks of a blob
  _BLOB_CHUNK_KIND_ = "__BlobChunk__"
  
  # Headers we ignore
  STRIPPED_HEADERS = frozenset(('content-length',
                                'content-md5',
                                'content-type',
                               ))
  
  UPLOAD_ERROR = 'There was an error with your upload. Redirect path not '\
    'found. Please contact the app owner if this persists.'
  
  # The maximum size of an incoming request.
  MAX_REQUEST_BUFF_SIZE = 2 * 1024 * 1024 * 1024  # 2GBs
  
  # The header used by GCS to provide a resumable upload ID.
  GCS_UPLOAD_ID_HEADER
  
  [... the rest is omitted ...]
files.old_path: AppDB/appscale/datastore/scripts/blobstore.py
files.new_contents: """ This service starts and stops application servers of a given application. """
  
  import fnmatch
  import glob
  import json
  import logging
  import math
  import os
  import psutil
  import shutil
  import SOAPpy
  import subprocess
  import sys
  import threading
  import time
  import urllib
  import urllib2
  from xml.etree import ElementTree
  
  from kazoo.client import KazooClient
  from M2Crypto import SSL
  from tornado.httpclient import HTTPClient
  from tornado.httpclient import HTTPError
  
  from appscale.common import (
    appscale_info,
    constants,
    file_io,
    monit_app_configuration,
    monit_interface,
    misc
  )
  from appscale.common.deployment_config import DeploymentConfig
  from appscale.common.deployment_config import ConfigInaccessible
  from appscale.common.monit_app_configuration import MONIT_CONFIG_DIR
  from appscale.common.monit_interface import MonitOperator
  from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
  
  sys.path.append(APPSCALE_PYTHON_APPSERVER)
  from google.appengine.api.appcontroller_client import AppControllerClient
  
  # The amount of seconds to wait for an application to start up.
  START_APP_TIMEOUT = 180
  
  # The amount of seconds to wait between checking if an application is up.
  BACKOFF_TIME = 1
  
  # The PID number to return when a process did not start correctly
  BAD_PID = -1
  
  # Default hourly cron directory.
  CRON_HOURLY = '/etc/cron.hourly'
  
  # Default logrotate configuration directory.
  LOGROTATE_CONFIG_DIR = '/etc/logrotate.d'
  
  # Max log size for AppScale Dashboard servers.
  DASHBOARD_LOG_SIZE = 10 * 1024 * 1024
  
  # Max application server log size in bytes.
  APP_LOG_SIZE = 250 * 1024 * 1024
  
  # Required configuration fields for starting an application
  REQUIRED_CONFIG_FIELDS = [
    'app_name',
    'app_port',
    'language',
    'login_ip',
    'env_vars',
    'max_memory']
  
  # The web path to fetch to see if the application is up
  FETCH_PATH = '/_ah/health_check'
  
  # The app ID of the AppScale Dashboard.
  APPSCALE_DASHBO
  
  [... the rest is omitted ...]
files.new_path: AppManager/app_manager_server.py
files.old_contents: """ This service starts and stops application servers of a given application. """
  
  import fnmatch
  import glob
  import json
  import logging
  import math
  import os
  import psutil
  import shutil
  import SOAPpy
  import subprocess
  import sys
  import threading
  import time
  import urllib
  import urllib2
  from xml.etree import ElementTree
  
  from M2Crypto import SSL
  from tornado.httpclient import HTTPClient
  from tornado.httpclient import HTTPError
  
  from appscale.common import (
    appscale_info,
    constants,
    file_io,
    monit_app_configuration,
    monit_interface,
    misc
  )
  from appscale.common.deployment_config import DeploymentConfig
  from appscale.common.deployment_config import ConfigInaccessible
  from appscale.common.monit_app_configuration import MONIT_CONFIG_DIR
  from appscale.common.monit_interface import MonitOperator
  from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
  
  sys.path.append(APPSCALE_PYTHON_APPSERVER)
  from google.appengine.api.appcontroller_client import AppControllerClient
  
  # The amount of seconds to wait for an application to start up.
  START_APP_TIMEOUT = 180
  
  # The amount of seconds to wait between checking if an application is up.
  BACKOFF_TIME = 1
  
  # The PID number to return when a process did not start correctly
  BAD_PID = -1
  
  # Default hourly cron directory.
  CRON_HOURLY = '/etc/cron.hourly'
  
  # Default logrotate configuration directory.
  LOGROTATE_CONFIG_DIR = '/etc/logrotate.d'
  
  # Max log size for AppScale Dashboard servers.
  DASHBOARD_LOG_SIZE = 10 * 1024 * 1024
  
  # Max application server log size in bytes.
  APP_LOG_SIZE = 250 * 1024 * 1024
  
  # Required configuration fields for starting an application
  REQUIRED_CONFIG_FIELDS = [
    'app_name',
    'app_port',
    'language',
    'login_ip',
    'env_vars',
    'max_memory']
  
  # The web path to fetch to see if the application is up
  FETCH_PATH = '/_ah/health_check'
  
  # The app ID of the AppScale Dashboard.
  APPSCALE_DASHBOARD_ID = "appscaledashboard"
  
  #
  
  [... the rest is omitted ...]
files.old_path: AppManager/app_manager_server.py
files.new_contents: import json
  import logging
  import time
  
  from kazoo.client import KazooException
  from kazoo.client import KazooState
  from kazoo.client import NoNodeError
  from kazoo.exceptions import ZookeeperError
  from threading import Lock
  
  from .constants import SMALL_WAIT
  from .constants import TINY_WAIT
  
  
  class ConfigInaccessible(Exception):
    """ Indicates that the configuration storage is not accessible. """
    pass
  
  
  class InvalidConfig(Exception):
    """ Indicates that a configuration option is invalid. """
    pass
  
  
  class ConfigStates(object):
    """ Possible states for DeploymentConfig. """
    LOADING = 'loading'
    LOADED = 'loaded'
  
  
  class DeploymentConfig(object):
    """ Accesses deployment configuration options. """
    # The ZooKeeper node where configuration is stored.
    CONFIG_ROOT = '/appscale/config'
  
    def __init__(self, zk_client):
      """ Creates new DeploymentConfig object.
  
      Args:
        zk_client: A KazooClient.
      """
      self.logger = logging.getLogger(self.__class__.__name__)
      self.update_lock = Lock()
      self.state = ConfigStates.LOADING
      self.config = {}
      self.conn = zk_client
      self.conn.add_listener(self._conn_listener)
      self.conn.ensure_path(self.CONFIG_ROOT)
      self.conn.ChildrenWatch(self.CONFIG_ROOT, func=self._update_config)
  
    def _conn_listener(self, state):
      """ Handles changes in ZooKeeper connection state.
  
      Args:
        state: A string indicating the new state.
      """
      if state == KazooState.LOST:
        self.logger.warning('ZK connection lost')
      if state == KazooState.SUSPENDED:
        self.logger.warning('ZK connection suspended')
      else:
        self.logger.info('ZK connection established')
  
    def _load_child(self, child):
      """ Fetches the data for a configuration node.
  
      Args:
        child: A string containing the ZooKeeper node to fetch.
      Returns:
        A dictionary containing configuration data.
      Raises:
        InaccessibleConfig if ZooKeeper is not accessible.
      """
      node = '/'.join([self.CONFIG_ROOT, child])
      try:
        data, _ = self.conn.ret
  
  [... the rest is omitted ...]
files.new_path: common/appscale/common/deployment_config.py
files.old_contents: import json
  import logging
  import time
  
  from kazoo.client import KazooClient
  from kazoo.client import KazooException
  from kazoo.client import KazooState
  from kazoo.client import NoNodeError
  from kazoo.exceptions import ZookeeperError
  from threading import Lock
  
  from .constants import SMALL_WAIT
  from .constants import TINY_WAIT
  
  
  class ConfigInaccessible(Exception):
    """ Indicates that the configuration storage is not accessible. """
    pass
  
  
  class InvalidConfig(Exception):
    """ Indicates that a configuration option is invalid. """
    pass
  
  
  class ConfigStates(object):
    """ Possible states for DeploymentConfig. """
    LOADING = 'loading'
    LOADED = 'loaded'
  
  
  class DeploymentConfig(object):
    """ Accesses deployment configuration options. """
    # The ZooKeeper node where configuration is stored.
    CONFIG_ROOT = '/appscale/config'
  
    def __init__(self, hosts):
      """ Creates new DeploymentConfig object.
  
      Args:
        hosts: A list of ZooKeeper hosts.
      """
      self.logger = logging.getLogger(self.__class__.__name__)
      self.update_lock = Lock()
      self.state = ConfigStates.LOADING
      self.config = {}
      self.conn = KazooClient(hosts=hosts, read_only=True)
      self.conn.add_listener(self._conn_listener)
      self.conn.start()
      self.conn.ensure_path(self.CONFIG_ROOT)
      self.conn.ChildrenWatch(self.CONFIG_ROOT, func=self._update_config)
  
    def _conn_listener(self, state):
      """ Handles changes in ZooKeeper connection state.
  
      Args:
        state: A string indicating the new state.
      """
      if state == KazooState.LOST:
        self.logger.warning('ZK connection lost')
      if state == KazooState.SUSPENDED:
        self.logger.warning('ZK connection suspended')
      else:
        self.logger.info('ZK connection established')
  
    def _load_child(self, child):
      """ Fetches the data for a configuration node.
  
      Args:
        child: A string containing the ZooKeeper node to fetch.
      Returns:
        A dictionary containing configuration data.
      Raises:
        InaccessibleConfig if ZooKeeper is not accessible.
      """
      node =
  
  [... the rest is omitted ...]
files.old_path: common/appscale/common/deployment_config.py
files.new_contents: #!/usr/bin/env python2
  """ This script writes all the configuration files necessary to start Cassandra
  on this machine."""
  
  import argparse
  import os
  import pkgutil
  import sys
  
  from kazoo.client import KazooClient
  
  from appscale.common import appscale_info
  from appscale.common.deployment_config import DeploymentConfig
  from appscale.common.deployment_config import InvalidConfig
  
  sys.path.append(os.path.join(os.path.dirname(__file__), '../AppDB'))
  from appscale.datastore.cassandra_env.cassandra_interface import\
    CASSANDRA_INSTALL_DIR
  
  
  if __name__ == "__main__":
    parser = argparse.ArgumentParser(
      description="Creates Cassandra's Monit configuration files")
    parser.add_argument('--local-ip', required=True,
                        help='The private IP address of this machine.')
    parser.add_argument('--master-ip', required=True,
                        help='The private IP address of the database master.')
    parser.add_argument('--zk-locations', required=False,
                        help='The location of Zookeeper.')
    args = parser.parse_args()
    zk_locations = args.zk_locations if args.zk_locations else \
      appscale_info.get_zk_locations_string()
    zk_client = KazooClient(hosts=zk_locations)
    zk_client.start()
    deployment_config = DeploymentConfig(zk_client)
    cassandra_config = deployment_config.get_config('cassandra')
    if 'num_tokens' not in cassandra_config:
      raise InvalidConfig('num_tokens not specified in deployment config.')
    num_tokens = cassandra_config['num_tokens']
  
    replacements = {'APPSCALE-LOCAL': args.local_ip,
                    'APPSCALE-MASTER': args.master_ip,
                    'APPSCALE-NUM-TOKENS': num_tokens}
  
    for filename in ('cassandra.yaml', 'cassandra-env.sh'):
      dest_file_path = os.path.join(CASSANDRA_INSTALL_DIR, 'cassandra', 'conf',
                                    filename)
      contents = pkgutil.get_data('appscale.datastore.cassandra_env',
                                  'templates/{}'.format(filename))
      for key, replacement in replacements.items():
        if replacement is None:
          replacement = ''
        contents = contents.replace(key, str(replacement))
     
  
  [... the rest is omitted ...]
files.new_path: scripts/setup_cassandra_config_files.py
files.old_contents: #!/usr/bin/env python2
  """ This script writes all the configuration files necessary to start Cassandra
  on this machine."""
  
  import argparse
  import os
  import pkgutil
  import sys
  
  from appscale.common import appscale_info
  from appscale.common.deployment_config import DeploymentConfig
  from appscale.common.deployment_config import InvalidConfig
  
  sys.path.append(os.path.join(os.path.dirname(__file__), '../AppDB'))
  from appscale.datastore.cassandra_env.cassandra_interface import\
    CASSANDRA_INSTALL_DIR
  
  
  if __name__ == "__main__":
    parser = argparse.ArgumentParser(
      description="Creates Cassandra's Monit configuration files")
    parser.add_argument('--local-ip', required=True,
                        help='The private IP address of this machine.')
    parser.add_argument('--master-ip', required=True,
                        help='The private IP address of the database master.')
    parser.add_argument('--zk-locations', required=False,
                        help='The location of Zookeeper.')
    args = parser.parse_args()
    zk_locations = args.zk_locations if args.zk_locations else \
      appscale_info.get_zk_locations_string()
    deployment_config = DeploymentConfig(zk_locations)
    cassandra_config = deployment_config.get_config('cassandra')
    if 'num_tokens' not in cassandra_config:
      raise InvalidConfig('num_tokens not specified in deployment config.')
    num_tokens = cassandra_config['num_tokens']
  
    replacements = {'APPSCALE-LOCAL': args.local_ip,
                    'APPSCALE-MASTER': args.master_ip,
                    'APPSCALE-NUM-TOKENS': num_tokens}
  
    for filename in ('cassandra.yaml', 'cassandra-env.sh'):
      dest_file_path = os.path.join(CASSANDRA_INSTALL_DIR, 'cassandra', 'conf',
                                    filename)
      contents = pkgutil.get_data('appscale.datastore.cassandra_env',
                                  'templates/{}'.format(filename))
      for key, replacement in replacements.items():
        if replacement is None:
          replacement = ''
        contents = contents.replace(key, str(replacement))
      with open(dest_file_path, 'w') as dest_file:
        dest_file.write(contents)
  
  
  [... the rest is omitted ...]
files.old_path: scripts/setup_cassandra_config_files.py