files: """
  Blobstore server for uploading blobs.
  See LICENSE file.
  
  Some code was taken from 
  http://blog.doughellmann.com/2009/07/pymotw-urllib2-library-for-opening-urls.html
  
  """
  import argparse
  import base64
  import cStringIO
  import datetime
  import gzip
  import hashlib
  import itertools
  import logging
  import math
  import mimetools
  import os 
  import os.path
  import requests
  import sys
  import tornado.httpserver
  import tornado.ioloop
  import tornado.web
  import urllib
  import urllib2
  
  from appscale.common import appscale_info
  from appscale.common.constants import LOG_FORMAT
  from appscale.common.deployment_config import DeploymentConfig
  from appscale.common.deployment_config import ConfigInaccessible
  from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
  from kazoo.client import KazooClient
  from StringIO import StringIO
  
  sys.path.append(APPSCALE_PYTHON_APPSERVER)
  from google.appengine.api import apiproxy_stub_map
  from google.appengine.api import datastore_errors
  from google.appengine.api import datastore_distributed
  from google.appengine.api import datastore
  from google.appengine.api.blobstore import blobstore
  from google.appengine.api.blobstore import datastore_blob_storage
  from google.appengine.tools import dev_appserver_upload
  
  # The URL path used for uploading blobs
  UPLOAD_URL_PATH = '_ah/upload/'
  
  # The port this service binds to
  DEFAULT_PORT = 6107
  
  # The datastore kind used for storing chunks of a blob
  _BLOB_CHUNK_KIND_ = "__BlobChunk__"
  
  # Headers we ignore
  STRIPPED_HEADERS = frozenset(('content-length',
                                'content-md5',
                                'content-type',
                               ))
  
  UPLOAD_ERROR = 'There was an error with your upload. Redirect path not '\
    'found. Please contact the app owner if this persists.'
  
  # The maximum size of an incoming request.
  MAX_REQUEST_BUFF_SIZE = 2 * 1024 * 1024 * 1024  # 2GBs
  
  # The header used by GCS to provide a resumable upload ID.
  GCS_UPLOAD_ID_HEADER = 'X-GUploader-UploadID'
  
  # The chunk size to use for uploading files to GCS.
  GCS_CHUNK_SIZE = 5 * 1024 * 1024  # 5MB
  
  # Global used for setting the datastore path when registering the DB
  datastore_path = ""
  
  # A DeploymentConfig accessor.
  deployment_config = None
  
  
  class MultiPartForm(object):
    """Accumulate the data to be used when posting a form."""
  
    def __init__(self, boundary):
      """ Constructor. """
      self.form_fields = []
      self.files = []
      if not boundary:
        self.boundary = mimetools.choose_boundary()
      else:
        self.boundary = boundary
      return
    
    def get_content_type(self):
      """ Get the content type to use. """
      return 'multipart/form-data; boundary=%s' % self.boundary
  
    def add_field(self, name, value):
      """Add a simple field to the form data."""
      self.form_fields.append((name, value))
      return
  
    def add_file(self, 
                 fieldname, 
                 filename,  
                 fileHandle, 
                 blob_key, 
                 access_type,
                 size,
                 creation):
      """Add a file to be uploaded."""
      body = fileHandle.read()
      mimetype = 'message/external-body; blob-key="%s"; access-type="%s"' % \
         (blob_key, access_type)
      self.files.append((fieldname, filename, mimetype, body, size, creation))
      return
    
    def __str__(self):
      """Return a string representing the form data, including attached files."""
      # Build a list of lists, each containing "lines" of the
      # request.  Each part is separated by a boundary string.
      # Once the list is built, return a string where each
      # line is separated by '\r\n'.  
      parts = []
      part_boundary = '--' + self.boundary
       
      # Add the form fields
      parts.extend(
          [ part_boundary,
            'Content-Disposition: form-data; name="%s"' % name,
            '',
            value,
          ]
          for name, value in self.form_fields
          )
       
      # Add the files to upload
      parts.extend(
          [ part_boundary,
            'Content-Type: %s' % content_type,
            'MIME-Version: 1.0',
            'Content-Disposition: form-data; name="%s"; filename="%s"' % \
               (field_name, filename),
            '',
            'Content-Type: application/octet-stream',
            'MIME-Version: 1.0',
            'Content-Length: %d'%size,
            'content-type: application/octet-stream',
            'content-disposition: form-data; name="%s"; filename="%s"' % \
               (field_name, filename),
            blobstore.UPLOAD_INFO_CREATION_HEADER + ": %s" % creation,
            '',
            ''
          ]
          for field_name, filename, content_type, body, size, \
              creation in self.files
          )
       
      # Flatten the list and add closing boundary marker,
      # then return CR+LF separated data
      flattened = list(itertools.chain(*parts))
      flattened.append('--' + self.boundary + '--')
      flattened.append('')
      return '\r\n'.join(flattened)
   
  def get_blobinfo(blob_key):
    """ Get BlobInfo from the datastore given its key. 
     
    Args:
      blob_key: A BlobInfo key
    Returns:
      The BlobInfo entity referenced by the blob_key.
    """
    datastore_key = datastore.Key.from_path(blobstore.BLOB_INFO_KIND,
                                            blob_key,
                                            namespace='')
    try:
      return datastore.Get(datastore_key)
    except datastore_errors.EntityNotFoundError:
      return None
  
  def get_session(session_id):
    """ Get the session entity of the given session ID. 
      
    Args:
      session_id: The session ID we want the cooresponding session entity.
    Returns:
      The session entity
    """
    try:
      return datastore.Get(session_id)
    except datastore_errors.EntityNotFoundError:
      return None
  
  def setup_env():
    """ Sets required environment variables for GAE datastore library """
    os.environ['AUTH_DOMAIN'] = "appscale.com"
    os.environ['USER_EMAIL'] = ""
    os.environ['USER_NICKNAME'] = ""
    os.environ['APPLICATION_ID'] = ""
  
  class Application(tornado.web.Application):
    """ The tornado web application handling uploads and healthchecks. """
    def __init__(self):
      """ Constructor. """
      handlers = [
        (r"/_ah/upload/(.*)/(.*)", UploadHandler),
        (r"/", HealthCheck)
      ]   
      tornado.web.Application.__init__(self, handlers)
  
  def split_content_type(c_type):
    """ Parses the content type. 
    
    Args: 
      c_type: the content type.
    Returns:
      A tuple the first portion of the content type and a key/value dictionary
      of the content-type.
    """
    delim = ';'
    ps = c_type.split(delim)
    tup = dict([(k.lower().strip(), v) for k, v in [p.split('=', 1) \
                                       for p in ps[1:]]])
    return tup
  
  class SmartRedirectHandler(urllib2.HTTPRedirectHandler):     
    """ An overridden class for custom actions on redirects. """
    def http_error_301(self, req, fp, code, msg, headers):  
      """ Stubbed out to do nothing since we do not follow redirects. """
      return None
  
    def http_error_302(self, req, fp, code, msg, headers):   
      """ Stubbed out to do nothing since we do not follow redirects. """
      return None
  
  class HealthCheck(tornado.web.RequestHandler):
    """ Tornado handler for health checks. """
    def get(self):
      """ This path is called to make sure the server is up and running. """
      self.finish("Hello") 
   
  class UploadHandler(tornado.web.RequestHandler):
    """ Tornado handler for uploads. """
    def post(self, app_id="blob", session_id = "session"):
      """ Handler a post request from a user uploading a blob. 
      
      Args:
        app_id: The application triggering the upload.
        session_id: Authentication token to validate the upload.
      """
      global datastore_path
      db = datastore_distributed.DatastoreDistributed(
        app_id, datastore_path, require_indexes=False)
      apiproxy_st
  
  [... the rest is omitted ...]
files: AppDB/appscale/datastore/scripts/blobstore.py
files: """
  Blobstore server for uploading blobs.
  See LICENSE file.
  
  Some code was taken from 
  http://blog.doughellmann.com/2009/07/pymotw-urllib2-library-for-opening-urls.html
  
  """
  import argparse
  import base64
  import cStringIO
  import datetime
  import gzip
  import hashlib
  import itertools
  import logging
  import math
  import mimetools
  import os 
  import os.path
  import requests
  import sys
  import tornado.httpserver
  import tornado.ioloop
  import tornado.web
  import urllib
  import urllib2
  
  from appscale.common import appscale_info
  from appscale.common.constants import LOG_FORMAT
  from appscale.common.deployment_config import DeploymentConfig
  from appscale.common.deployment_config import ConfigInaccessible
  from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
  from StringIO import StringIO
  
  sys.path.append(APPSCALE_PYTHON_APPSERVER)
  from google.appengine.api import apiproxy_stub_map
  from google.appengine.api import datastore_errors
  from google.appengine.api import datastore_distributed
  from google.appengine.api import datastore
  from google.appengine.api.blobstore import blobstore
  from google.appengine.api.blobstore import datastore_blob_storage
  from google.appengine.tools import dev_appserver_upload
  
  # The URL path used for uploading blobs
  UPLOAD_URL_PATH = '_ah/upload/'
  
  # The port this service binds to
  DEFAULT_PORT = 6107
  
  # The datastore kind used for storing chunks of a blob
  _BLOB_CHUNK_KIND_ = "__BlobChunk__"
  
  # Headers we ignore
  STRIPPED_HEADERS = frozenset(('content-length',
                                'content-md5',
                                'content-type',
                               ))
  
  UPLOAD_ERROR = 'There was an error with your upload. Redirect path not '\
    'found. Please contact the app owner if this persists.'
  
  # The maximum size of an incoming request.
  MAX_REQUEST_BUFF_SIZE = 2 * 1024 * 1024 * 1024  # 2GBs
  
  # The header used by GCS to provide a resumable upload ID.
  GCS_UPLOAD_ID_HEADER = 'X-GUploader-UploadID'
  
  # The chunk size to use for uploading files to GCS.
  GCS_CHUNK_SIZE = 5 * 1024 * 1024  # 5MB
  
  # Global used for setting the datastore path when registering the DB
  datastore_path = ""
  
  # A DeploymentConfig accessor.
  deployment_config = None
  
  
  class MultiPartForm(object):
    """Accumulate the data to be used when posting a form."""
  
    def __init__(self, boundary):
      """ Constructor. """
      self.form_fields = []
      self.files = []
      if not boundary:
        self.boundary = mimetools.choose_boundary()
      else:
        self.boundary = boundary
      return
    
    def get_content_type(self):
      """ Get the content type to use. """
      return 'multipart/form-data; boundary=%s' % self.boundary
  
    def add_field(self, name, value):
      """Add a simple field to the form data."""
      self.form_fields.append((name, value))
      return
  
    def add_file(self, 
                 fieldname, 
                 filename,  
                 fileHandle, 
                 blob_key, 
                 access_type,
                 size,
                 creation):
      """Add a file to be uploaded."""
      body = fileHandle.read()
      mimetype = 'message/external-body; blob-key="%s"; access-type="%s"' % \
         (blob_key, access_type)
      self.files.append((fieldname, filename, mimetype, body, size, creation))
      return
    
    def __str__(self):
      """Return a string representing the form data, including attached files."""
      # Build a list of lists, each containing "lines" of the
      # request.  Each part is separated by a boundary string.
      # Once the list is built, return a string where each
      # line is separated by '\r\n'.  
      parts = []
      part_boundary = '--' + self.boundary
       
      # Add the form fields
      parts.extend(
          [ part_boundary,
            'Content-Disposition: form-data; name="%s"' % name,
            '',
            value,
          ]
          for name, value in self.form_fields
          )
       
      # Add the files to upload
      parts.extend(
          [ part_boundary,
            'Content-Type: %s' % content_type,
            'MIME-Version: 1.0',
            'Content-Disposition: form-data; name="%s"; filename="%s"' % \
               (field_name, filename),
            '',
            'Content-Type: application/octet-stream',
            'MIME-Version: 1.0',
            'Content-Length: %d'%size,
            'content-type: application/octet-stream',
            'content-disposition: form-data; name="%s"; filename="%s"' % \
               (field_name, filename),
            blobstore.UPLOAD_INFO_CREATION_HEADER + ": %s" % creation,
            '',
            ''
          ]
          for field_name, filename, content_type, body, size, \
              creation in self.files
          )
       
      # Flatten the list and add closing boundary marker,
      # then return CR+LF separated data
      flattened = list(itertools.chain(*parts))
      flattened.append('--' + self.boundary + '--')
      flattened.append('')
      return '\r\n'.join(flattened)
   
  def get_blobinfo(blob_key):
    """ Get BlobInfo from the datastore given its key. 
     
    Args:
      blob_key: A BlobInfo key
    Returns:
      The BlobInfo entity referenced by the blob_key.
    """
    datastore_key = datastore.Key.from_path(blobstore.BLOB_INFO_KIND,
                                            blob_key,
                                            namespace='')
    try:
      return datastore.Get(datastore_key)
    except datastore_errors.EntityNotFoundError:
      return None
  
  def get_session(session_id):
    """ Get the session entity of the given session ID. 
      
    Args:
      session_id: The session ID we want the cooresponding session entity.
    Returns:
      The session entity
    """
    try:
      return datastore.Get(session_id)
    except datastore_errors.EntityNotFoundError:
      return None
  
  def setup_env():
    """ Sets required environment variables for GAE datastore library """
    os.environ['AUTH_DOMAIN'] = "appscale.com"
    os.environ['USER_EMAIL'] = ""
    os.environ['USER_NICKNAME'] = ""
    os.environ['APPLICATION_ID'] = ""
  
  class Application(tornado.web.Application):
    """ The tornado web application handling uploads and healthchecks. """
    def __init__(self):
      """ Constructor. """
      handlers = [
        (r"/_ah/upload/(.*)/(.*)", UploadHandler),
        (r"/", HealthCheck)
      ]   
      tornado.web.Application.__init__(self, handlers)
  
  def split_content_type(c_type):
    """ Parses the content type. 
    
    Args: 
      c_type: the content type.
    Returns:
      A tuple the first portion of the content type and a key/value dictionary
      of the content-type.
    """
    delim = ';'
    ps = c_type.split(delim)
    tup = dict([(k.lower().strip(), v) for k, v in [p.split('=', 1) \
                                       for p in ps[1:]]])
    return tup
  
  class SmartRedirectHandler(urllib2.HTTPRedirectHandler):     
    """ An overridden class for custom actions on redirects. """
    def http_error_301(self, req, fp, code, msg, headers):  
      """ Stubbed out to do nothing since we do not follow redirects. """
      return None
  
    def http_error_302(self, req, fp, code, msg, headers):   
      """ Stubbed out to do nothing since we do not follow redirects. """
      return None
  
  class HealthCheck(tornado.web.RequestHandler):
    """ Tornado handler for health checks. """
    def get(self):
      """ This path is called to make sure the server is up and running. """
      self.finish("Hello") 
   
  class UploadHandler(tornado.web.RequestHandler):
    """ Tornado handler for uploads. """
    def post(self, app_id="blob", session_id = "session"):
      """ Handler a post request from a user uploading a blob. 
      
      Args:
        app_id: The application triggering the upload.
        session_id: Authentication token to validate the upload.
      """
      global datastore_path
      db = datastore_distributed.DatastoreDistributed(
        app_id, datastore_path, require_indexes=False)
      apiproxy_stub_map.apiproxy.RegisterStub('dat
  
  [... the rest is omitted ...]
files: AppDB/appscale/datastore/scripts/blobstore.py
files: """ This service starts and stops application servers of a given application. """
  
  import fnmatch
  import glob
  import json
  import logging
  import math
  import os
  import psutil
  import shutil
  import SOAPpy
  import subprocess
  import sys
  import threading
  import time
  import urllib
  import urllib2
  from xml.etree import ElementTree
  
  from kazoo.client import KazooClient
  from M2Crypto import SSL
  from tornado.httpclient import HTTPClient
  from tornado.httpclient import HTTPError
  
  from appscale.common import (
    appscale_info,
    constants,
    file_io,
    monit_app_configuration,
    monit_interface,
    misc
  )
  from appscale.common.deployment_config import DeploymentConfig
  from appscale.common.deployment_config import ConfigInaccessible
  from appscale.common.monit_app_configuration import MONIT_CONFIG_DIR
  from appscale.common.monit_interface import MonitOperator
  from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
  
  sys.path.append(APPSCALE_PYTHON_APPSERVER)
  from google.appengine.api.appcontroller_client import AppControllerClient
  
  # The amount of seconds to wait for an application to start up.
  START_APP_TIMEOUT = 180
  
  # The amount of seconds to wait between checking if an application is up.
  BACKOFF_TIME = 1
  
  # The PID number to return when a process did not start correctly
  BAD_PID = -1
  
  # Default hourly cron directory.
  CRON_HOURLY = '/etc/cron.hourly'
  
  # Default logrotate configuration directory.
  LOGROTATE_CONFIG_DIR = '/etc/logrotate.d'
  
  # Max log size for AppScale Dashboard servers.
  DASHBOARD_LOG_SIZE = 10 * 1024 * 1024
  
  # Max application server log size in bytes.
  APP_LOG_SIZE = 250 * 1024 * 1024
  
  # Required configuration fields for starting an application
  REQUIRED_CONFIG_FIELDS = [
    'app_name',
    'app_port',
    'language',
    'login_ip',
    'env_vars',
    'max_memory']
  
  # The web path to fetch to see if the application is up
  FETCH_PATH = '/_ah/health_check'
  
  # The app ID of the AppScale Dashboard.
  APPSCALE_DASHBOARD_ID = "appscaledashboard"
  
  # Apps which can access any application's data.
  TRUSTED_APPS = ["appscaledashboard"]
  
  # The flag to tell the application server that this application can access
  # all application data.
  TRUSTED_FLAG = "--trusted"
  
  # The location on the filesystem where the PHP executable is installed.
  PHP_CGI_LOCATION = "/usr/bin/php-cgi"
  
  # The location of the App Engine SDK for Go.
  GO_SDK = os.path.join('/', 'opt', 'go_appengine')
  
  HTTP_OK = 200
  
  # The amount of seconds to wait before retrying to add routing.
  ROUTING_RETRY_INTERVAL = 5
  
  PIDFILE_TEMPLATE = os.path.join('/', 'var', 'run', 'appscale',
                                  'app___{project}-{port}.pid')
  
  # The number of seconds an instance is allowed to finish serving requests after
  # it receives a shutdown signal.
  MAX_INSTANCE_RESPONSE_TIME = 600
  
  # A DeploymentConfig accessor.
  deployment_config = None
  
  # An interface for working with Monit.
  monit_operator = MonitOperator()
  
  
  class BadConfigurationException(Exception):
    """ An application is configured incorrectly. """
    def __init__(self, value):
      Exception.__init__(self, value)
      self.value = value
  
    def __str__(self):
      return repr(self.value)
  
  class NoRedirection(urllib2.HTTPErrorProcessor):
    """ A url opener that does not automatically redirect. """
    def http_response(self, request, response):
      """ Processes HTTP responses.
  
      Args:
        request: An HTTP request object.
        response: An HTTP response object.
      Returns:
        The HTTP response object.
      """
      return response
    https_response = http_response
  
  def convert_config_from_json(config):
    """ Takes the configuration in JSON format and converts it to a dictionary.
        Validates the dictionary configuration before returning.
  
    Args:
      config: The configuration to convert
    Returns:
      None if it failed to convert the config and a dictionary if it succeeded
    """
    logging.info("Configuration for app:" + str(config))
    try:
      config = json.loads(config)
    except ValueError, error:
      logging.error("%s Exception--Unable to parse configuration: %s" % \
        (error.__class__, str(error)))
      return None
    except TypeError, error:
      logging.error("%s Exception--Unable to parse configuration: %s" % \
        (error.__class__, str(error)))
      return None
  
    if is_config_valid(config):
      return config
    else:
      return None
  
  def add_routing(app, port):
    """ Tells the AppController to begin routing traffic to an AppServer.
  
    Args:
      app: A string that contains the application ID.
      port: A string that contains the port that the AppServer listens on.
    """
    logging.info("Waiting for application {} on port {} to be active.".
      format(str(app), str(port)))
    if not wait_on_app(port):
      # In case the AppServer fails we let the AppController to detect it
      # and remove it if it still show in monit.
      logging.warning("AppServer did not come up in time, for {}:{}.".
        format(str(app), str(port)))
      return
  
    acc = appscale_info.get_appcontroller_client()
    appserver_ip = appscale_info.get_private_ip()
  
    while True:
      result = acc.add_routing_for_appserver(app, appserver_ip, port)
      if result == AppControllerClient.NOT_READY:
        logging.info('AppController not yet ready to add routing.')
        time.sleep(ROUTING_RETRY_INTERVAL)
      else:
        break
  
    logging.info('Successfully established routing for {} on port {}'.
      format(app, port))
  
  def start_app(config):
    """ Starts a Google App Engine application on this machine. It
        will start it up and then proceed to fetch the main page.
  
    Args:
      config: a dictionary that contains
         app_name: Name of the application to start
         app_port: Port to start on
         language: What language the app is written in
         login_ip: Public ip of deployment
         env_vars: A dict of environment variables that should be passed to the
          app.
         max_memory: An int that names the maximum amount of memory that this
          App Engine app is allowed to consume before being restarted.
         syslog_server: The IP of the syslog server to send the application
           logs to. Usually it's the login private IP.
    Returns:
      PID of process on success, -1 otherwise
    """
    config = convert_config_from_json(config)
    if config is None:
      logging.error("Invalid configuration for application")
      return BAD_PID
  
    if not misc.is_app_name_valid(config['app_name']):
      logging.error("Invalid app name for application: " + config['app_name'])
      return BAD_PID
    logging.info("Starting %s application %s" % (
      config['language'], config['app_name']))
  
    env_vars = config['env_vars']
    pidfile = PIDFILE_TEMPLATE.format(project=config['app_name'],
                                      port=config['app_port'])
  
    if config['language'] == constants.GO:
      env_vars['GOPATH'] = os.path.join('/var', 'apps', config['app_name'],
                                        'gopath')
      env_vars['GOROOT'] = os.path.join(GO_SDK, 'goroot')
  
    watch = "app___" + config['app_name']
    match_cmd = ""
  
    if config['language'] == constants.PYTHON27 or \
        config['language'] == constants.GO or \
        config['language'] == constants.PHP:
      start_cmd = create_python27_start_cmd(
        config['app_name'],
        config['login_ip'],
        config['app_port'],
        pidfile)
      stop_cmd = create_python27_stop_cmd(config['app_port'])
      env_vars.update(create_python_app_env(
        config['login_ip'],
        config['app_name']))
    elif config['language'] == constants.JAVA:
      remove_conflicting_jars(config['app_name'])
      copy_successful = copy_modified_jars(config['app_name'])
      if not copy_successful:
        return BAD_PID
  
      # Account for MaxPermSize (~170MB), the parent process (~50MB), and thread
      # stacks (~20MB).
      max_heap = config['max_memory'] - 250
      if max_heap <= 0:
        return BAD_PID
      start_cmd = create_java_start_cmd(
        config['app_name'],
        config['app_port'],
        config['login_ip'],
        max_heap,
        pidfile
      )
      match_cmd = "java -ea -cp.*--port={}.*{}".format(str(config
  
  [... the rest is omitted ...]
files: AppManager/app_manager_server.py
files: """ This service starts and stops application servers of a given application. """
  
  import fnmatch
  import glob
  import json
  import logging
  import math
  import os
  import psutil
  import shutil
  import SOAPpy
  import subprocess
  import sys
  import threading
  import time
  import urllib
  import urllib2
  from xml.etree import ElementTree
  
  from M2Crypto import SSL
  from tornado.httpclient import HTTPClient
  from tornado.httpclient import HTTPError
  
  from appscale.common import (
    appscale_info,
    constants,
    file_io,
    monit_app_configuration,
    monit_interface,
    misc
  )
  from appscale.common.deployment_config import DeploymentConfig
  from appscale.common.deployment_config import ConfigInaccessible
  from appscale.common.monit_app_configuration import MONIT_CONFIG_DIR
  from appscale.common.monit_interface import MonitOperator
  from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
  
  sys.path.append(APPSCALE_PYTHON_APPSERVER)
  from google.appengine.api.appcontroller_client import AppControllerClient
  
  # The amount of seconds to wait for an application to start up.
  START_APP_TIMEOUT = 180
  
  # The amount of seconds to wait between checking if an application is up.
  BACKOFF_TIME = 1
  
  # The PID number to return when a process did not start correctly
  BAD_PID = -1
  
  # Default hourly cron directory.
  CRON_HOURLY = '/etc/cron.hourly'
  
  # Default logrotate configuration directory.
  LOGROTATE_CONFIG_DIR = '/etc/logrotate.d'
  
  # Max log size for AppScale Dashboard servers.
  DASHBOARD_LOG_SIZE = 10 * 1024 * 1024
  
  # Max application server log size in bytes.
  APP_LOG_SIZE = 250 * 1024 * 1024
  
  # Required configuration fields for starting an application
  REQUIRED_CONFIG_FIELDS = [
    'app_name',
    'app_port',
    'language',
    'login_ip',
    'env_vars',
    'max_memory']
  
  # The web path to fetch to see if the application is up
  FETCH_PATH = '/_ah/health_check'
  
  # The app ID of the AppScale Dashboard.
  APPSCALE_DASHBOARD_ID = "appscaledashboard"
  
  # Apps which can access any application's data.
  TRUSTED_APPS = ["appscaledashboard"]
  
  # The flag to tell the application server that this application can access
  # all application data.
  TRUSTED_FLAG = "--trusted"
  
  # The location on the filesystem where the PHP executable is installed.
  PHP_CGI_LOCATION = "/usr/bin/php-cgi"
  
  # The location of the App Engine SDK for Go.
  GO_SDK = os.path.join('/', 'opt', 'go_appengine')
  
  HTTP_OK = 200
  
  # The amount of seconds to wait before retrying to add routing.
  ROUTING_RETRY_INTERVAL = 5
  
  PIDFILE_TEMPLATE = os.path.join('/', 'var', 'run', 'appscale',
                                  'app___{project}-{port}.pid')
  
  # The number of seconds an instance is allowed to finish serving requests after
  # it receives a shutdown signal.
  MAX_INSTANCE_RESPONSE_TIME = 600
  
  # A DeploymentConfig accessor.
  deployment_config = None
  
  # An interface for working with Monit.
  monit_operator = MonitOperator()
  
  
  class BadConfigurationException(Exception):
    """ An application is configured incorrectly. """
    def __init__(self, value):
      Exception.__init__(self, value)
      self.value = value
  
    def __str__(self):
      return repr(self.value)
  
  class NoRedirection(urllib2.HTTPErrorProcessor):
    """ A url opener that does not automatically redirect. """
    def http_response(self, request, response):
      """ Processes HTTP responses.
  
      Args:
        request: An HTTP request object.
        response: An HTTP response object.
      Returns:
        The HTTP response object.
      """
      return response
    https_response = http_response
  
  def convert_config_from_json(config):
    """ Takes the configuration in JSON format and converts it to a dictionary.
        Validates the dictionary configuration before returning.
  
    Args:
      config: The configuration to convert
    Returns:
      None if it failed to convert the config and a dictionary if it succeeded
    """
    logging.info("Configuration for app:" + str(config))
    try:
      config = json.loads(config)
    except ValueError, error:
      logging.error("%s Exception--Unable to parse configuration: %s" % \
        (error.__class__, str(error)))
      return None
    except TypeError, error:
      logging.error("%s Exception--Unable to parse configuration: %s" % \
        (error.__class__, str(error)))
      return None
  
    if is_config_valid(config):
      return config
    else:
      return None
  
  def add_routing(app, port):
    """ Tells the AppController to begin routing traffic to an AppServer.
  
    Args:
      app: A string that contains the application ID.
      port: A string that contains the port that the AppServer listens on.
    """
    logging.info("Waiting for application {} on port {} to be active.".
      format(str(app), str(port)))
    if not wait_on_app(port):
      # In case the AppServer fails we let the AppController to detect it
      # and remove it if it still show in monit.
      logging.warning("AppServer did not come up in time, for {}:{}.".
        format(str(app), str(port)))
      return
  
    acc = appscale_info.get_appcontroller_client()
    appserver_ip = appscale_info.get_private_ip()
  
    while True:
      result = acc.add_routing_for_appserver(app, appserver_ip, port)
      if result == AppControllerClient.NOT_READY:
        logging.info('AppController not yet ready to add routing.')
        time.sleep(ROUTING_RETRY_INTERVAL)
      else:
        break
  
    logging.info('Successfully established routing for {} on port {}'.
      format(app, port))
  
  def start_app(config):
    """ Starts a Google App Engine application on this machine. It
        will start it up and then proceed to fetch the main page.
  
    Args:
      config: a dictionary that contains
         app_name: Name of the application to start
         app_port: Port to start on
         language: What language the app is written in
         login_ip: Public ip of deployment
         env_vars: A dict of environment variables that should be passed to the
          app.
         max_memory: An int that names the maximum amount of memory that this
          App Engine app is allowed to consume before being restarted.
         syslog_server: The IP of the syslog server to send the application
           logs to. Usually it's the login private IP.
    Returns:
      PID of process on success, -1 otherwise
    """
    config = convert_config_from_json(config)
    if config is None:
      logging.error("Invalid configuration for application")
      return BAD_PID
  
    if not misc.is_app_name_valid(config['app_name']):
      logging.error("Invalid app name for application: " + config['app_name'])
      return BAD_PID
    logging.info("Starting %s application %s" % (
      config['language'], config['app_name']))
  
    env_vars = config['env_vars']
    pidfile = PIDFILE_TEMPLATE.format(project=config['app_name'],
                                      port=config['app_port'])
  
    if config['language'] == constants.GO:
      env_vars['GOPATH'] = os.path.join('/var', 'apps', config['app_name'],
                                        'gopath')
      env_vars['GOROOT'] = os.path.join(GO_SDK, 'goroot')
  
    watch = "app___" + config['app_name']
    match_cmd = ""
  
    if config['language'] == constants.PYTHON27 or \
        config['language'] == constants.GO or \
        config['language'] == constants.PHP:
      start_cmd = create_python27_start_cmd(
        config['app_name'],
        config['login_ip'],
        config['app_port'],
        pidfile)
      stop_cmd = create_python27_stop_cmd(config['app_port'])
      env_vars.update(create_python_app_env(
        config['login_ip'],
        config['app_name']))
    elif config['language'] == constants.JAVA:
      remove_conflicting_jars(config['app_name'])
      copy_successful = copy_modified_jars(config['app_name'])
      if not copy_successful:
        return BAD_PID
  
      # Account for MaxPermSize (~170MB), the parent process (~50MB), and thread
      # stacks (~20MB).
      max_heap = config['max_memory'] - 250
      if max_heap <= 0:
        return BAD_PID
      start_cmd = create_java_start_cmd(
        config['app_name'],
        config['app_port'],
        config['login_ip'],
        max_heap,
        pidfile
      )
      match_cmd = "java -ea -cp.*--port={}.*{}".format(str(config['app_port']),
        os.path.dirname
  
  [... the rest is omitted ...]
files: AppManager/app_manager_server.py
files: import json
  import logging
  import time
  
  from kazoo.client import KazooException
  from kazoo.client import KazooState
  from kazoo.client import NoNodeError
  from kazoo.exceptions import ZookeeperError
  from threading import Lock
  
  from .constants import SMALL_WAIT
  from .constants import TINY_WAIT
  
  
  class ConfigInaccessible(Exception):
    """ Indicates that the configuration storage is not accessible. """
    pass
  
  
  class InvalidConfig(Exception):
    """ Indicates that a configuration option is invalid. """
    pass
  
  
  class ConfigStates(object):
    """ Possible states for DeploymentConfig. """
    LOADING = 'loading'
    LOADED = 'loaded'
  
  
  class DeploymentConfig(object):
    """ Accesses deployment configuration options. """
    # The ZooKeeper node where configuration is stored.
    CONFIG_ROOT = '/appscale/config'
  
    def __init__(self, zk_client):
      """ Creates new DeploymentConfig object.
  
      Args:
        zk_client: A KazooClient.
      """
      self.logger = logging.getLogger(self.__class__.__name__)
      self.update_lock = Lock()
      self.state = ConfigStates.LOADING
      self.config = {}
      self.conn = zk_client
      self.conn.add_listener(self._conn_listener)
      self.conn.ensure_path(self.CONFIG_ROOT)
      self.conn.ChildrenWatch(self.CONFIG_ROOT, func=self._update_config)
  
    def _conn_listener(self, state):
      """ Handles changes in ZooKeeper connection state.
  
      Args:
        state: A string indicating the new state.
      """
      if state == KazooState.LOST:
        self.logger.warning('ZK connection lost')
      if state == KazooState.SUSPENDED:
        self.logger.warning('ZK connection suspended')
      else:
        self.logger.info('ZK connection established')
  
    def _load_child(self, child):
      """ Fetches the data for a configuration node.
  
      Args:
        child: A string containing the ZooKeeper node to fetch.
      Returns:
        A dictionary containing configuration data.
      Raises:
        InaccessibleConfig if ZooKeeper is not accessible.
      """
      node = '/'.join([self.CONFIG_ROOT, child])
      try:
        data, _ = self.conn.retry(self.conn.get, node)
      except (KazooException, ZookeeperError):
        raise ConfigInaccessible('ZooKeeper connection not available')
      except NoNodeError:
        return {}
  
      try:
        return json.loads(data)
      except ValueError:
        self.logger.warning('Invalid deployment config: {}'.format(child))
        return {}
  
    def _update_config(self, children):
      """ Updates configuration when it changes.
  
      Args:
        children: A list of ZooKeeper nodes.
      """
      with self.update_lock:
        self.state = ConfigStates.LOADING
  
        # Ensure old sections are removed.
        self.config = {}
  
        for child in children:
          while True:
            try:
              self.config[child] = self._load_child(child)
              break
            except ConfigInaccessible as load_error:
              self.logger.warning(str(load_error))
              time.sleep(SMALL_WAIT)
  
        self.logger.info('Deployment configuration updated')
        self.state = ConfigStates.LOADED
  
    def get_config(self, section):
      """ Fetches the configuration for a given section.
  
      Args:
        section: A string specifying the section to fetch.
      Returns:
        A dictionary containing configuration data.
      Raises:
        InaccessibleConfig if ZooKeeper is inaccessible.
      """
      # If the connection is established, it should finish loading very soon.
      while (self.state == ConfigStates.LOADING and
             self.conn.state not in (KazooState.LOST, KazooState.SUSPENDED)):
        time.sleep(TINY_WAIT)
  
      if self.state != ConfigStates.LOADED:
        raise ConfigInaccessible('ZooKeeper connection not available')
  
      with self.update_lock:
        if section not in self.config:
          return {}
        return self.config[section]
  
    def close(self):
      """ Close the ZooKeeper connection. """
      self.conn.stop()
  
  
  [... the rest is omitted ...]
files: common/appscale/common/deployment_config.py
files: import json
  import logging
  import time
  
  from kazoo.client import KazooClient
  from kazoo.client import KazooException
  from kazoo.client import KazooState
  from kazoo.client import NoNodeError
  from kazoo.exceptions import ZookeeperError
  from threading import Lock
  
  from .constants import SMALL_WAIT
  from .constants import TINY_WAIT
  
  
  class ConfigInaccessible(Exception):
    """ Indicates that the configuration storage is not accessible. """
    pass
  
  
  class InvalidConfig(Exception):
    """ Indicates that a configuration option is invalid. """
    pass
  
  
  class ConfigStates(object):
    """ Possible states for DeploymentConfig. """
    LOADING = 'loading'
    LOADED = 'loaded'
  
  
  class DeploymentConfig(object):
    """ Accesses deployment configuration options. """
    # The ZooKeeper node where configuration is stored.
    CONFIG_ROOT = '/appscale/config'
  
    def __init__(self, hosts):
      """ Creates new DeploymentConfig object.
  
      Args:
        hosts: A list of ZooKeeper hosts.
      """
      self.logger = logging.getLogger(self.__class__.__name__)
      self.update_lock = Lock()
      self.state = ConfigStates.LOADING
      self.config = {}
      self.conn = KazooClient(hosts=hosts, read_only=True)
      self.conn.add_listener(self._conn_listener)
      self.conn.start()
      self.conn.ensure_path(self.CONFIG_ROOT)
      self.conn.ChildrenWatch(self.CONFIG_ROOT, func=self._update_config)
  
    def _conn_listener(self, state):
      """ Handles changes in ZooKeeper connection state.
  
      Args:
        state: A string indicating the new state.
      """
      if state == KazooState.LOST:
        self.logger.warning('ZK connection lost')
      if state == KazooState.SUSPENDED:
        self.logger.warning('ZK connection suspended')
      else:
        self.logger.info('ZK connection established')
  
    def _load_child(self, child):
      """ Fetches the data for a configuration node.
  
      Args:
        child: A string containing the ZooKeeper node to fetch.
      Returns:
        A dictionary containing configuration data.
      Raises:
        InaccessibleConfig if ZooKeeper is not accessible.
      """
      node = '/'.join([self.CONFIG_ROOT, child])
      try:
        data, _ = self.conn.retry(self.conn.get, node)
      except (KazooException, ZookeeperError):
        raise ConfigInaccessible('ZooKeeper connection not available')
      except NoNodeError:
        return {}
  
      try:
        return json.loads(data)
      except ValueError:
        self.logger.warning('Invalid deployment config: {}'.format(child))
        return {}
  
    def _update_config(self, children):
      """ Updates configuration when it changes.
  
      Args:
        children: A list of ZooKeeper nodes.
      """
      with self.update_lock:
        self.state = ConfigStates.LOADING
  
        # Ensure old sections are removed.
        self.config = {}
  
        for child in children:
          while True:
            try:
              self.config[child] = self._load_child(child)
              break
            except ConfigInaccessible as load_error:
              self.logger.warning(str(load_error))
              time.sleep(SMALL_WAIT)
  
        self.logger.info('Deployment configuration updated')
        self.state = ConfigStates.LOADED
  
    def get_config(self, section):
      """ Fetches the configuration for a given section.
  
      Args:
        section: A string specifying the section to fetch.
      Returns:
        A dictionary containing configuration data.
      Raises:
        InaccessibleConfig if ZooKeeper is inaccessible.
      """
      # If the connection is established, it should finish loading very soon.
      while (self.state == ConfigStates.LOADING and
             self.conn.state not in (KazooState.LOST, KazooState.SUSPENDED)):
        time.sleep(TINY_WAIT)
  
      if self.state != ConfigStates.LOADED:
        raise ConfigInaccessible('ZooKeeper connection not available')
  
      with self.update_lock:
        if section not in self.config:
          return {}
        return self.config[section]
  
    def close(self):
      """ Close the ZooKeeper connection. """
      self.conn.stop()
  
  
  [... the rest is omitted ...]
files: common/appscale/common/deployment_config.py
files: #!/usr/bin/env python2
  """ This script writes all the configuration files necessary to start Cassandra
  on this machine."""
  
  import argparse
  import os
  import pkgutil
  import sys
  
  from kazoo.client import KazooClient
  
  from appscale.common import appscale_info
  from appscale.common.deployment_config import DeploymentConfig
  from appscale.common.deployment_config import InvalidConfig
  
  sys.path.append(os.path.join(os.path.dirname(__file__), '../AppDB'))
  from appscale.datastore.cassandra_env.cassandra_interface import\
    CASSANDRA_INSTALL_DIR
  
  
  if __name__ == "__main__":
    parser = argparse.ArgumentParser(
      description="Creates Cassandra's Monit configuration files")
    parser.add_argument('--local-ip', required=True,
                        help='The private IP address of this machine.')
    parser.add_argument('--master-ip', required=True,
                        help='The private IP address of the database master.')
    parser.add_argument('--zk-locations', required=False,
                        help='The location of Zookeeper.')
    args = parser.parse_args()
    zk_locations = args.zk_locations if args.zk_locations else \
      appscale_info.get_zk_locations_string()
    zk_client = KazooClient(hosts=zk_locations)
    zk_client.start()
    deployment_config = DeploymentConfig(zk_client)
    cassandra_config = deployment_config.get_config('cassandra')
    if 'num_tokens' not in cassandra_config:
      raise InvalidConfig('num_tokens not specified in deployment config.')
    num_tokens = cassandra_config['num_tokens']
  
    replacements = {'APPSCALE-LOCAL': args.local_ip,
                    'APPSCALE-MASTER': args.master_ip,
                    'APPSCALE-NUM-TOKENS': num_tokens}
  
    for filename in ('cassandra.yaml', 'cassandra-env.sh'):
      dest_file_path = os.path.join(CASSANDRA_INSTALL_DIR, 'cassandra', 'conf',
                                    filename)
      contents = pkgutil.get_data('appscale.datastore.cassandra_env',
                                  'templates/{}'.format(filename))
      for key, replacement in replacements.items():
        if replacement is None:
          replacement = ''
        contents = contents.replace(key, str(replacement))
      with open(dest_file_path, 'w') as dest_file:
        dest_file.write(contents)
  
  
  [... the rest is omitted ...]
files: scripts/setup_cassandra_config_files.py
files: #!/usr/bin/env python2
  """ This script writes all the configuration files necessary to start Cassandra
  on this machine."""
  
  import argparse
  import os
  import pkgutil
  import sys
  
  from appscale.common import appscale_info
  from appscale.common.deployment_config import DeploymentConfig
  from appscale.common.deployment_config import InvalidConfig
  
  sys.path.append(os.path.join(os.path.dirname(__file__), '../AppDB'))
  from appscale.datastore.cassandra_env.cassandra_interface import\
    CASSANDRA_INSTALL_DIR
  
  
  if __name__ == "__main__":
    parser = argparse.ArgumentParser(
      description="Creates Cassandra's Monit configuration files")
    parser.add_argument('--local-ip', required=True,
                        help='The private IP address of this machine.')
    parser.add_argument('--master-ip', required=True,
                        help='The private IP address of the database master.')
    parser.add_argument('--zk-locations', required=False,
                        help='The location of Zookeeper.')
    args = parser.parse_args()
    zk_locations = args.zk_locations if args.zk_locations else \
      appscale_info.get_zk_locations_string()
    deployment_config = DeploymentConfig(zk_locations)
    cassandra_config = deployment_config.get_config('cassandra')
    if 'num_tokens' not in cassandra_config:
      raise InvalidConfig('num_tokens not specified in deployment config.')
    num_tokens = cassandra_config['num_tokens']
  
    replacements = {'APPSCALE-LOCAL': args.local_ip,
                    'APPSCALE-MASTER': args.master_ip,
                    'APPSCALE-NUM-TOKENS': num_tokens}
  
    for filename in ('cassandra.yaml', 'cassandra-env.sh'):
      dest_file_path = os.path.join(CASSANDRA_INSTALL_DIR, 'cassandra', 'conf',
                                    filename)
      contents = pkgutil.get_data('appscale.datastore.cassandra_env',
                                  'templates/{}'.format(filename))
      for key, replacement in replacements.items():
        if replacement is None:
          replacement = ''
        contents = contents.replace(key, str(replacement))
      with open(dest_file_path, 'w') as dest_file:
        dest_file.write(contents)
  
  
  [... the rest is omitted ...]
files: scripts/setup_cassandra_config_files.py