files: """
  Blobstore server for uploading blobs.
  See LICENSE file.
  
  Some code was taken from 
  http://blog.doughellmann.com/2009/07/pymotw-urllib2-library-for-opening-urls.html
  
  """
  import argparse
  import base64
  import cStringIO
  import datetime
  import gzip
  import hashlib
  import itertools
  import logging
  import math
  import mimetools
  import os 
  import os.path
  import requests
  import sys
  import tornado.httpserver
  import tornado.ioloop
  import tornado.web
  import urllib
  import urllib2
  
  from appscale.common import appscale_info
  from appscale.common.constants import LOG_FORMAT
  from appscale.common.deployment_config import DeploymentConfig
  from appscale.common.deployment_config import ConfigInaccessible
  from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
  from kazoo.client import KazooClient
  from StringIO import StringIO
  
  sys.path.append(APPSCALE_PYTHON_APPSERVER)
  from google.appengine.api import apiproxy_stub_map
  from google.appengine.api import datastore_errors
  from google.appengine.api import datastore_distributed
  from google.appengine.api import datastore
  from google.appengine.api.blobstore import blobstore
  from google.appengine.api.blobstore import datastore_blob_storage
  from google.appengine.tools import dev_appserver_upload
  
  # The URL path used for uploading blobs
  UPLOAD_URL_PATH = '_ah/upload/'
  
  # The port this service binds to
  DEFAULT_PORT = 6107
  
  # The datastore kind used for storing chunks of a blob
  _BLOB_CHUNK_KIND_ = "__BlobChunk__"
  
  # Headers we ignore
  STRIPPED_HEADERS = frozenset(('content-length',
                                'content-md5',
                                'content-type',
                               ))
  
  UPLOAD_ERROR = 'There was an error with your upload. Redirect path not '\
    'found. Please contact the app owner if this persists.'
  
  # The maximum size of an incoming request.
  MAX_REQUEST_BUFF_SIZE = 2 * 1024 * 1024 * 1024  # 2GBs
  
  # The header used by GCS to provide a resumable upload ID.
  GCS_UPLOAD_ID_HEADER = 'X-GUploader-UploadID'
  
  # The chunk size to use for uploading files to GCS.
  GCS_CHUNK_SIZE = 5 * 1024 * 1024  # 5MB
  
  # Global used for setting the datastore path when registering the DB
  datastore_path = ""
  
  # A DeploymentConfig accessor.
  deployment_config = None
  
  
  class MultiPartForm(object):
    """Accumulate the data to be used when posting a form."""
  
    def __init__(self, boundary):
      """ Constructor. """
      self.form_fields = []
      self.files = []
      if not boundary:
        self.boundary = mimetools.choose_boundary()
      else:
        self.boundary = boundary
      return
    
    def get_content_type(self):
      """ Get the content type to use. """
      return 'multipart/form-data; boundary=%s' % self.boundary
  
    def add_field(self, name, value):
      """Add a simple field to the form data."""
      self.form_fields.append((name, value))
      return
  
    def add_file(self, 
                 fieldname, 
                 filename,  
                 fileHandle, 
                 blob_key, 
                 access_type,
                 size,
                 creation):
      """Add a file to be uploaded."""
      body = fileHandle.read()
      mimetype = 'message/external-body; blob-key="%s"; access-type="%s"' % \
         (blob_key, access_type)
      self.files.append((fieldname, filename, mimetype, body, size, creation))
      return
    
    def __str__(self):
      """Return a string representing the form data, including attached files."""
      # Build a list of lists, each containing "lines" of the
      # request.  Each part is separated by a boundary string.
      # Once the list is built, return a string where each
      # line is separated by '\r\n'.  
      parts = []
      part_boundary = '--' + self.boundary
       
      # Add the form fields
      parts.extend(
          [ part_boundary,
            'Content-Disposition: form-data; name="%s"' % name,
            '',
            value,
          ]
          for name, value in self.form_fields
          )
       
      # Add the files to upload
      parts.extend(
          [ part_boundary,
            'Content-Type: %s' % content_type,
            'MIME-Version: 1.0',
            'Content-Disposition: form-data; name="%s"; filename="%s"' % \
               (field_name, filename),
            '',
            'Content-Type: application/octet-stream',
            'MIME-Version: 1.0',
            'Content-Length: %d'%size,
            'content-type: application/octet-stream',
            'content-disposition: form-data; name="%s"; filename="%s"' % \
               (field_name, filename),
            blobstore.UPLOAD_INFO_CREATION_HEADER + ": %s" % creation,
            '',
            ''
          ]
          for field_name, filename, content_type, body, size, \
              creation in self.files
          )
       
      # Flatten the list and add closing boundary marker,
      # then return CR+LF separated data
      flattened = list(itertools.chain(*parts))
      flattened.append('--' + self.boundary + '--')
      flattened.append('')
      return '\r\n'.join(flattened)
   
  def get_blobinfo(blob_key):
    """ Get BlobInfo from the datastore given its key. 
     
    Args:
      blob_key: A BlobInfo key
    Returns:
      The BlobInfo entity referenced by the blob_key.
    """
    datastore_key = datastore.Key.from_path(blobstore.BLOB_INFO_KIND,
                                            blob_key,
                                            namespace='')
    try:
      return datastore.Get(datastore_key)
    except datastore_errors.EntityNotFoundError:
      return None
  
  def get_session(session_id):
    """ Get the session entity of the given session ID. 
      
    Args:
      session_id: The session ID we want the cooresponding session entity.
    Returns:
      The session entity
    """
    try:
      return datastore.Get(session_id)
    except datastore_errors.EntityNotFoundError:
      return None
  
  def setup_env():
    """ Sets required environment variables for GAE datastore library """
    os.environ['AUTH_DOMAIN'] = "appscale.com"
    os.environ['USER_EMAIL'] = ""
    os.environ['USER_NICKNAME'] = ""
    os.environ['APPLICATION_ID'] = ""
  
  class Application(tornado.web.Application):
    """ The tornado web application handling uploads and healthchecks. """
    def __init__(self):
      """ Constructor. """
      handlers = [
        (r"/_ah/upload/(.*)/(.*)", UploadHandler),
        (r"/", HealthCheck)
      ]   
      tornado.web.Application.__init__(self, handlers)
  
  def split_content_type(c_type):
    """ Parses the content type. 
    
    Args: 
      c_type: the content type.
    Returns:
      A tuple the first portion of the content type and a key/value dictionary
      of the content-type.
    """
    delim = ';'
    ps = c_type.split(delim)
    tup = dict([(k.lower().strip(), v) for k, v in [p.split('=', 1) \
                                       for p in ps[1:]]])
    return tup
  
  class SmartRedirectHandler(urllib2.HTTPRedirectHandler):     
    """ An overridden class for custom actions on redirects. """
    def http_error_301(self, req, fp, code, msg, headers):  
      """ Stubbed out to do nothing since we do not follow redirects. """
      return None
  
    def http_error_302(self, req, fp, code, msg, headers):   
      """ Stubbed out to do nothing since we do not follow redirects. """
      return None
  
  class HealthCheck(tornado.web.RequestHandler):
    """ Tornado handler for health checks. """
    def get(self):
      """ This path is called to make sure the server is up and running. """
      self.finish("Hello") 
   
  class UploadHandler(tornado.web.RequestHandler):
    """ Tornado handler for uploads. """
    def post(self, app_id="blob", session_id = "session"):
      """ Handler a post request from a user uploading a blob. 
      
      Args:
        app_id: The application triggering the upload.
        session_id: Authentication token to validate the upload.
      """
      global datastore_path
      db = datastore_distributed.DatastoreDistributed(
        app_id, datastore_path, require_indexes=False)
      apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', db)
      os.environ['APPLICATION_ID'] = app_id
  
      # Setup the app id in the datastore.
      # Get session info and upload success path.
      blob_session = get_session(session_id)
      if not blob_session:
        self.finish('Session has expired. Contact the owner of the ' + \
                    'app for support.\n\n')
        return
  
      success_path = blob_session["success_path"]
  
      server_host = success_path[:success_path.rfind("/", 3)]
      if server_host.startswith("http://"):
        # Strip off the beginging of the server host
        server_host = server_host[len("http://"):]
      server_host = server_host.split('/')[0]
  
      blob_storage = datastore_blob_storage.DatastoreBlobStorage(app_id)
      uploadhandler = dev_appserver_upload.UploadCGIHandler(blob_storage)
  
      datastore.Delete(blob_session)
  
      # This request is sent to the upload handler of the app
      # in the hope it returns a redirect to be forwarded to the user
      urlrequest = urllib2.Request(success_path)
  
      # Forward all relevant headers and create data for request
      content_type = self.request.headers["Content-Type"]
      kv = split_content_type(content_type)
      boundary = None
      if "boundary" in kv:
        boundary = kv["boundary"]
  
      urlrequest.add_header("Content-Type",
                            'application/x-www-form-urlencoded')
  
      for name, value in self.request.headers.items():
        if name.lower() not in STRIPPED_HEADERS:
          urlrequest.add_header(name, value)
    
      # Get correct redirect addresses, otherwise it will redirect back
      # to this port.
      urlrequest.add_header("Host", server_host)
  
      form = MultiPartForm(boundary)
      creation = datetime.datetime.now()
  
      # Loop on all files in the form.
      for filekey in self.request.files.keys():
        data = {"blob_info_metadata": {filekey: []}}
        file = self.request.files[filekey][0] 
        body = file["body"]
        size = len(body)
        filename = file["filename"]
        file_content_type = file["content_type"]
  
        gs_path = ''
        if 'gcs_bucket' in blob_session:
          gcs_config = {'scheme': 'https', 'port': 443}
          try:
            gcs_config.update(deployment_config.get_config('gcs'))
          except ConfigInaccessible:
            self.send_error('Unable to fetch GCS configuration.')
            return
  
          if 'host' not in gcs_config:
            self.send_error('GCS host is not defined.')
            return
  
          gcs_path = '{scheme}://{host}:{port}'.format(**gcs_config)
          gcs_bucket_name = blob_session['gcs_bucket']
          gcs_url = '/'.join([gcs_path, gcs_bucket_name, filename])
          response = requests.post(gcs_url,
                                   headers={'x-goog-resumable': 'start'})
          if (response.status_code != 201 or
              GCS_UPLOAD_ID_HEADER not in response.headers):
            self.send_error(reason='Unable to start resumable GCS upload.')
            return
          upload_id = response.headers[GCS_UPLOAD_ID_HEADER]
  
          total_chunks = int(math.ceil(float(size) / GCS_CHUNK_SIZE))
          for chunk_num in range(total_chunks):
            offset = GCS_CHUNK_SIZE * chunk_num
            current_chunk_size = min(GCS_CHUNK_SIZE, size - offset)
            end_byte = offset + current_chunk_size
            current_range = '{}-{}'.format(offset, end_byte - 1)
            content_range = 'bytes {}/{}'.format(current_range, size)
            response = requests.put(gcs_url, data=body[offset:end_byte],
                                    headers={'Content-Range': content_range},
                                    params={'upload_id': upload_id})
            if chunk_num == total_chunks - 1:
              if response.status_code != 200:
                self.send_error(reason='Unable to complete GCS upload.')
                return
            else:
              if response.status_code != 308:
                self.send_error(reason='Unable to continue GCS upload.')
                return
          gs_path = '/gs/{}/{}'.format(gcs_bucket_name, filename)
          blob_key = 'encoded_gs_key:' + base64.b64encode(gs_path)
        else:
          blob_entity = uploadhandler.StoreBlob(file, creation)
          blob_key = str(blob_entity.key().name())
  
        if not blob_key: 
          self.finish('Status: 500\n\n')
          return 
        creation_formatted = blobstore._format_creation(creation)
        form.add_file(filekey, filename, cStringIO.StringIO(blob_key), blob_key,
                      blobstore.BLOB_KEY_HEADER, size, creation_formatted)
  
        md5_handler = hashlib.md5(str(body))
        blob_info = {"filename": filename,
                     "creation-date": creation_formatted,
                     "key": blob_key,
                     "size": str(size),
                     "content-type": file_content_type,
                     "md5-hash": md5_handler.hexdigest()}
        if 'gcs_bucket' in blob_session:
          blob_info['gs-name'] = gs_path
        data["blob_info_metadata"][filekey].append(blob_info)
  
      # Loop through form fields
      for fieldkey in self.request.arguments.keys():
        form.add_field(fieldkey, self.request.arguments[fieldkey][0])
        data[fieldkey] = self.request.arguments[fieldkey][0]
  
      logging.debug("Callback data: \n{}".format(data))
      data = urllib.urlencode(data)
      urlrequest.add_header("Content-Length", str(len(data)))
      urlrequest.add_data(data)
  
      # We are catching the redirect error here
      # and extracting the Location to post the redirect.
      try:
        response = urllib2.urlopen(urlrequest)
        output = response.read()
        if response.info().get('Content-Encoding') == 'gzip':
          buf = StringIO(output)
          f = gzip.GzipFile(fileobj=buf)
          data = f.read()
          output = data
        self.finish(output)
      except urllib2.HTTPError, e: 
        if "Location" in e.hdrs:
          # Catch any errors, use the success path to 
          # get the ip and port, use the redirect path
          # for the path. We split redirect_path just in case
          # its a full path.
          redirect_path = e.hdrs["Location"]
          self.redirect(redirect_path)
          return
        else:
          self.finish(UPLOAD_ERROR + "</br>" + str(e.hdrs) + "</br>" + str(e))
          return
  
  
  def main():
    global datastore_path
    global deployment_config
  
    logging.basicConfig(format=LOG_FORMAT, level=logging.INFO)
  
    parser = argparse.ArgumentParser()
    parser.add_argument('-p', '--port', type=int, default=DEFAULT_PORT,
                        required=True, help="The blobstore server's port")
    parser.add_argument('-d', '--datastore-path', required=True,
                        help='The location of the datastore server')
    args = parser.parse_args()
  
    datastore_path = args.datastore_path
    zk_ips = appscale_info.get_zk_node_ips()
    zk_client = KazooClient(hosts=','.join(zk_ips))
    zk_client.start()
    deployment_config = DeploymentConfig(zk_client)
    setup_env()
  
    http_server = tornado.httpserver.HTTPServer(
      Application(), max_buffer_size=MAX_REQUEST_BUFF_SIZE)
  
    http_server.listen(args.port)
  
    acc = appscale_info.get_appcontroller_client()
    acc.add_routing_for_blob_server()
  
    logging.info('Starting BlobServer on {}'.format(args.port))
    tornado.ioloop.IOLoop.instance().start()
  
files: AppDB/appscale/datastore/scripts/blobstore.py
files: """
  Blobstore server for uploading blobs.
  See LICENSE file.
  
  Some code was taken from 
  http://blog.doughellmann.com/2009/07/pymotw-urllib2-library-for-opening-urls.html
  
  """
  import argparse
  import base64
  import cStringIO
  import datetime
  import gzip
  import hashlib
  import itertools
  import logging
  import math
  import mimetools
  import os 
  import os.path
  import requests
  import sys
  import tornado.httpserver
  import tornado.ioloop
  import tornado.web
  import urllib
  import urllib2
  
  from appscale.common import appscale_info
  from appscale.common.constants import LOG_FORMAT
  from appscale.common.deployment_config import DeploymentConfig
  from appscale.common.deployment_config import ConfigInaccessible
  from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
  from StringIO import StringIO
  
  sys.path.append(APPSCALE_PYTHON_APPSERVER)
  from google.appengine.api import apiproxy_stub_map
  from google.appengine.api import datastore_errors
  from google.appengine.api import datastore_distributed
  from google.appengine.api import datastore
  from google.appengine.api.blobstore import blobstore
  from google.appengine.api.blobstore import datastore_blob_storage
  from google.appengine.tools import dev_appserver_upload
  
  # The URL path used for uploading blobs
  UPLOAD_URL_PATH = '_ah/upload/'
  
  # The port this service binds to
  DEFAULT_PORT = 6107
  
  # The datastore kind used for storing chunks of a blob
  _BLOB_CHUNK_KIND_ = "__BlobChunk__"
  
  # Headers we ignore
  STRIPPED_HEADERS = frozenset(('content-length',
                                'content-md5',
                                'content-type',
                               ))
  
  UPLOAD_ERROR = 'There was an error with your upload. Redirect path not '\
    'found. Please contact the app owner if this persists.'
  
  # The maximum size of an incoming request.
  MAX_REQUEST_BUFF_SIZE = 2 * 1024 * 1024 * 1024  # 2GBs
  
  # The header used by GCS to provide a resumable upload ID.
  GCS_UPLOAD_ID_HEADER = 'X-GUploader-UploadID'
  
  # The chunk size to use for uploading files to GCS.
  GCS_CHUNK_SIZE = 5 * 1024 * 1024  # 5MB
  
  # Global used for setting the datastore path when registering the DB
  datastore_path = ""
  
  # A DeploymentConfig accessor.
  deployment_config = None
  
  
  class MultiPartForm(object):
    """Accumulate the data to be used when posting a form."""
  
    def __init__(self, boundary):
      """ Constructor. """
      self.form_fields = []
      self.files = []
      if not boundary:
        self.boundary = mimetools.choose_boundary()
      else:
        self.boundary = boundary
      return
    
    def get_content_type(self):
      """ Get the content type to use. """
      return 'multipart/form-data; boundary=%s' % self.boundary
  
    def add_field(self, name, value):
      """Add a simple field to the form data."""
      self.form_fields.append((name, value))
      return
  
    def add_file(self, 
                 fieldname, 
                 filename,  
                 fileHandle, 
                 blob_key, 
                 access_type,
                 size,
                 creation):
      """Add a file to be uploaded."""
      body = fileHandle.read()
      mimetype = 'message/external-body; blob-key="%s"; access-type="%s"' % \
         (blob_key, access_type)
      self.files.append((fieldname, filename, mimetype, body, size, creation))
      return
    
    def __str__(self):
      """Return a string representing the form data, including attached files."""
      # Build a list of lists, each containing "lines" of the
      # request.  Each part is separated by a boundary string.
      # Once the list is built, return a string where each
      # line is separated by '\r\n'.  
      parts = []
      part_boundary = '--' + self.boundary
       
      # Add the form fields
      parts.extend(
          [ part_boundary,
            'Content-Disposition: form-data; name="%s"' % name,
            '',
            value,
          ]
          for name, value in self.form_fields
          )
       
      # Add the files to upload
      parts.extend(
          [ part_boundary,
            'Content-Type: %s' % content_type,
            'MIME-Version: 1.0',
            'Content-Disposition: form-data; name="%s"; filename="%s"' % \
               (field_name, filename),
            '',
            'Content-Type: application/octet-stream',
            'MIME-Version: 1.0',
            'Content-Length: %d'%size,
            'content-type: application/octet-stream',
            'content-disposition: form-data; name="%s"; filename="%s"' % \
               (field_name, filename),
            blobstore.UPLOAD_INFO_CREATION_HEADER + ": %s" % creation,
            '',
            ''
          ]
          for field_name, filename, content_type, body, size, \
              creation in self.files
          )
       
      # Flatten the list and add closing boundary marker,
      # then return CR+LF separated data
      flattened = list(itertools.chain(*parts))
      flattened.append('--' + self.boundary + '--')
      flattened.append('')
      return '\r\n'.join(flattened)
   
  def get_blobinfo(blob_key):
    """ Get BlobInfo from the datastore given its key. 
     
    Args:
      blob_key: A BlobInfo key
    Returns:
      The BlobInfo entity referenced by the blob_key.
    """
    datastore_key = datastore.Key.from_path(blobstore.BLOB_INFO_KIND,
                                            blob_key,
                                            namespace='')
    try:
      return datastore.Get(datastore_key)
    except datastore_errors.EntityNotFoundError:
      return None
  
  def get_session(session_id):
    """ Get the session entity of the given session ID. 
      
    Args:
      session_id: The session ID we want the cooresponding session entity.
    Returns:
      The session entity
    """
    try:
      return datastore.Get(session_id)
    except datastore_errors.EntityNotFoundError:
      return None
  
  def setup_env():
    """ Sets required environment variables for GAE datastore library """
    os.environ['AUTH_DOMAIN'] = "appscale.com"
    os.environ['USER_EMAIL'] = ""
    os.environ['USER_NICKNAME'] = ""
    os.environ['APPLICATION_ID'] = ""
  
  class Application(tornado.web.Application):
    """ The tornado web application handling uploads and healthchecks. """
    def __init__(self):
      """ Constructor. """
      handlers = [
        (r"/_ah/upload/(.*)/(.*)", UploadHandler),
        (r"/", HealthCheck)
      ]   
      tornado.web.Application.__init__(self, handlers)
  
  def split_content_type(c_type):
    """ Parses the content type. 
    
    Args: 
      c_type: the content type.
    Returns:
      A tuple the first portion of the content type and a key/value dictionary
      of the content-type.
    """
    delim = ';'
    ps = c_type.split(delim)
    tup = dict([(k.lower().strip(), v) for k, v in [p.split('=', 1) \
                                       for p in ps[1:]]])
    return tup
  
  class SmartRedirectHandler(urllib2.HTTPRedirectHandler):     
    """ An overridden class for custom actions on redirects. """
    def http_error_301(self, req, fp, code, msg, headers):  
      """ Stubbed out to do nothing since we do not follow redirects. """
      return None
  
    def http_error_302(self, req, fp, code, msg, headers):   
      """ Stubbed out to do nothing since we do not follow redirects. """
      return None
  
  class HealthCheck(tornado.web.RequestHandler):
    """ Tornado handler for health checks. """
    def get(self):
      """ This path is called to make sure the server is up and running. """
      self.finish("Hello") 
   
  class UploadHandler(tornado.web.RequestHandler):
    """ Tornado handler for uploads. """
    def post(self, app_id="blob", session_id = "session"):
      """ Handler a post request from a user uploading a blob. 
      
      Args:
        app_id: The application triggering the upload.
        session_id: Authentication token to validate the upload.
      """
      global datastore_path
      db = datastore_distributed.DatastoreDistributed(
        app_id, datastore_path, require_indexes=False)
      apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', db)
      os.environ['APPLICATION_ID'] = app_id
  
      # Setup the app id in the datastore.
      # Get session info and upload success path.
      blob_session = get_session(session_id)
      if not blob_session:
        self.finish('Session has expired. Contact the owner of the ' + \
                    'app for support.\n\n')
        return
  
      success_path = blob_session["success_path"]
  
      server_host = success_path[:success_path.rfind("/", 3)]
      if server_host.startswith("http://"):
        # Strip off the beginging of the server host
        server_host = server_host[len("http://"):]
      server_host = server_host.split('/')[0]
  
      blob_storage = datastore_blob_storage.DatastoreBlobStorage(app_id)
      uploadhandler = dev_appserver_upload.UploadCGIHandler(blob_storage)
  
      datastore.Delete(blob_session)
  
      # This request is sent to the upload handler of the app
      # in the hope it returns a redirect to be forwarded to the user
      urlrequest = urllib2.Request(success_path)
  
      # Forward all relevant headers and create data for request
      content_type = self.request.headers["Content-Type"]
      kv = split_content_type(content_type)
      boundary = None
      if "boundary" in kv:
        boundary = kv["boundary"]
  
      urlrequest.add_header("Content-Type",
                            'application/x-www-form-urlencoded')
  
      for name, value in self.request.headers.items():
        if name.lower() not in STRIPPED_HEADERS:
          urlrequest.add_header(name, value)
    
      # Get correct redirect addresses, otherwise it will redirect back
      # to this port.
      urlrequest.add_header("Host", server_host)
  
      form = MultiPartForm(boundary)
      creation = datetime.datetime.now()
  
      # Loop on all files in the form.
      for filekey in self.request.files.keys():
        data = {"blob_info_metadata": {filekey: []}}
        file = self.request.files[filekey][0] 
        body = file["body"]
        size = len(body)
        filename = file["filename"]
        file_content_type = file["content_type"]
  
        gs_path = ''
        if 'gcs_bucket' in blob_session:
          gcs_config = {'scheme': 'https', 'port': 443}
          try:
            gcs_config.update(deployment_config.get_config('gcs'))
          except ConfigInaccessible:
            self.send_error('Unable to fetch GCS configuration.')
            return
  
          if 'host' not in gcs_config:
            self.send_error('GCS host is not defined.')
            return
  
          gcs_path = '{scheme}://{host}:{port}'.format(**gcs_config)
          gcs_bucket_name = blob_session['gcs_bucket']
          gcs_url = '/'.join([gcs_path, gcs_bucket_name, filename])
          response = requests.post(gcs_url,
                                   headers={'x-goog-resumable': 'start'})
          if (response.status_code != 201 or
              GCS_UPLOAD_ID_HEADER not in response.headers):
            self.send_error(reason='Unable to start resumable GCS upload.')
            return
          upload_id = response.headers[GCS_UPLOAD_ID_HEADER]
  
          total_chunks = int(math.ceil(float(size) / GCS_CHUNK_SIZE))
          for chunk_num in range(total_chunks):
            offset = GCS_CHUNK_SIZE * chunk_num
            current_chunk_size = min(GCS_CHUNK_SIZE, size - offset)
            end_byte = offset + current_chunk_size
            current_range = '{}-{}'.format(offset, end_byte - 1)
            content_range = 'bytes {}/{}'.format(current_range, size)
            response = requests.put(gcs_url, data=body[offset:end_byte],
                                    headers={'Content-Range': content_range},
                                    params={'upload_id': upload_id})
            if chunk_num == total_chunks - 1:
              if response.status_code != 200:
                self.send_error(reason='Unable to complete GCS upload.')
                return
            else:
              if response.status_code != 308:
                self.send_error(reason='Unable to continue GCS upload.')
                return
          gs_path = '/gs/{}/{}'.format(gcs_bucket_name, filename)
          blob_key = 'encoded_gs_key:' + base64.b64encode(gs_path)
        else:
          blob_entity = uploadhandler.StoreBlob(file, creation)
          blob_key = str(blob_entity.key().name())
  
        if not blob_key: 
          self.finish('Status: 500\n\n')
          return 
        creation_formatted = blobstore._format_creation(creation)
        form.add_file(filekey, filename, cStringIO.StringIO(blob_key), blob_key,
                      blobstore.BLOB_KEY_HEADER, size, creation_formatted)
  
        md5_handler = hashlib.md5(str(body))
        blob_info = {"filename": filename,
                     "creation-date": creation_formatted,
                     "key": blob_key,
                     "size": str(size),
                     "content-type": file_content_type,
                     "md5-hash": md5_handler.hexdigest()}
        if 'gcs_bucket' in blob_session:
          blob_info['gs-name'] = gs_path
        data["blob_info_metadata"][filekey].append(blob_info)
  
      # Loop through form fields
      for fieldkey in self.request.arguments.keys():
        form.add_field(fieldkey, self.request.arguments[fieldkey][0])
        data[fieldkey] = self.request.arguments[fieldkey][0]
  
      logging.debug("Callback data: \n{}".format(data))
      data = urllib.urlencode(data)
      urlrequest.add_header("Content-Length", str(len(data)))
      urlrequest.add_data(data)
  
      # We are catching the redirect error here
      # and extracting the Location to post the redirect.
      try:
        response = urllib2.urlopen(urlrequest)
        output = response.read()
        if response.info().get('Content-Encoding') == 'gzip':
          buf = StringIO(output)
          f = gzip.GzipFile(fileobj=buf)
          data = f.read()
          output = data
        self.finish(output)
      except urllib2.HTTPError, e: 
        if "Location" in e.hdrs:
          # Catch any errors, use the success path to 
          # get the ip and port, use the redirect path
          # for the path. We split redirect_path just in case
          # its a full path.
          redirect_path = e.hdrs["Location"]
          self.redirect(redirect_path)
          return
        else:
          self.finish(UPLOAD_ERROR + "</br>" + str(e.hdrs) + "</br>" + str(e))
          return
  
  
  def main():
    global datastore_path
    global deployment_config
  
    logging.basicConfig(format=LOG_FORMAT, level=logging.INFO)
  
    parser = argparse.ArgumentParser()
    parser.add_argument('-p', '--port', type=int, default=DEFAULT_PORT,
                        required=True, help="The blobstore server's port")
    parser.add_argument('-d', '--datastore-path', required=True,
                        help='The location of the datastore server')
    args = parser.parse_args()
  
    datastore_path = args.datastore_path
    deployment_config = DeploymentConfig(appscale_info.get_zk_locations_string())
    setup_env()
  
    http_server = tornado.httpserver.HTTPServer(
      Application(), max_buffer_size=MAX_REQUEST_BUFF_SIZE)
  
    http_server.listen(args.port)
  
    acc = appscale_info.get_appcontroller_client()
    acc.add_routing_for_blob_server()
  
    logging.info('Starting BlobServer on {}'.format(args.port))
    tornado.ioloop.IOLoop.instance().start()
  
files: AppDB/appscale/datastore/scripts/blobstore.py
files: """ This service starts and stops application servers of a given application. """
  
  import fnmatch
  import glob
  import json
  import logging
  import math
  import os
  import psutil
  import shutil
  import SOAPpy
  import subprocess
  import sys
  import threading
  import time
  import urllib
  import urllib2
  from xml.etree import ElementTree
  
  from kazoo.client import KazooClient
  from M2Crypto import SSL
  from tornado.httpclient import HTTPClient
  from tornado.httpclient import HTTPError
  
  from appscale.common import (
    appscale_info,
    constants,
    file_io,
    monit_app_configuration,
    monit_interface,
    misc
  )
  from appscale.common.deployment_config import DeploymentConfig
  from appscale.common.deployment_config import ConfigInaccessible
  from appscale.common.monit_app_configuration import MONIT_CONFIG_DIR
  from appscale.common.monit_interface import MonitOperator
  from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
  
  sys.path.append(APPSCALE_PYTHON_APPSERVER)
  from google.appengine.api.appcontroller_client import AppControllerClient
  
  # The amount of seconds to wait for an application to start up.
  START_APP_TIMEOUT = 180
  
  # The amount of seconds to wait between checking if an application is up.
  BACKOFF_TIME = 1
  
  # The PID number to return when a process did not start correctly
  BAD_PID = -1
  
  # Default hourly cron directory.
  CRON_HOURLY = '/etc/cron.hourly'
  
  # Default logrotate configuration directory.
  LOGROTATE_CONFIG_DIR = '/etc/logrotate.d'
  
  # Max log size for AppScale Dashboard servers.
  DASHBOARD_LOG_SIZE = 10 * 1024 * 1024
  
  # Max application server log size in bytes.
  APP_LOG_SIZE = 250 * 1024 * 1024
  
  # Required configuration fields for starting an application
  REQUIRED_CONFIG_FIELDS = [
    'app_name',
    'app_port',
    'language',
    'login_ip',
    'env_vars',
    'max_memory']
  
  # The web path to fetch to see if the application is up
  FETCH_PATH = '/_ah/health_check'
  
  # The app ID of the AppScale Dashboard.
  APPSCALE_DASHBOARD_ID = "appscaledashboard"
  
  # Apps which can access any application's data.
  TRUSTED_APPS = ["appscaledashboard"]
  
  # The flag to tell the application server that this application can access
  # all application data.
  TRUSTED_FLAG = "--trusted"
  
  # The location on the filesystem where the PHP executable is installed.
  PHP_CGI_LOCATION = "/usr/bin/php-cgi"
  
  # The location of the App Engine SDK for Go.
  GO_SDK = os.path.join('/', 'opt', 'go_appengine')
  
  HTTP_OK = 200
  
  # The amount of seconds to wait before retrying to add routing.
  ROUTING_RETRY_INTERVAL = 5
  
  PIDFILE_TEMPLATE = os.path.join('/', 'var', 'run', 'appscale',
                                  'app___{project}-{port}.pid')
  
  # The number of seconds an instance is allowed to finish serving requests after
  # it receives a shutdown signal.
  MAX_INSTANCE_RESPONSE_TIME = 600
  
  # A DeploymentConfig accessor.
  deployment_config = None
  
  # An interface for working with Monit.
  monit_operator = MonitOperator()
  
  
  class BadConfigurationException(Exception):
    """ An application is configured incorrectly. """
    def __init__(self, value):
      Exception.__init__(self, value)
      self.value = value
  
    def __str__(self):
      return repr(self.value)
  
  class NoRedirection(urllib2.HTTPErrorProcessor):
    """ A url opener that does not automatically redirect. """
    def http_response(self, request, response):
      """ Processes HTTP responses.
  
      Args:
        request: An HTTP request object.
        response: An HTTP response object.
      Returns:
        The HTTP response object.
      """
      return response
    https_response = http_response
  
  def convert_config_from_json(config):
    """ Takes the configuration in JSON format and converts it to a dictionary.
        Validates the dictionary configuration before returning.
  
    Args:
      config: The configuration to convert
    Returns:
      None if it failed to convert the config and a dictionary if it succeeded
    """
    logging.info("Configuration for app:" + str(config))
    try:
      config = json.loads(config)
    except ValueError, error:
      logging.error("%s Exception--Unable to parse configuration: %s" % \
        (error.__class__, str(error)))
      return None
    except TypeError, error:
      logging.error("%s Exception--Unable to parse configuration: %s" % \
        (error.__class__, str(error)))
      return None
  
    if is_config_valid(config):
      return config
    else:
      return None
  
  def add_routing(app, port):
    """ Tells the AppController to begin routing traffic to an AppServer.
  
    Args:
      app: A string that contains the application ID.
      port: A string that contains the port that the AppServer listens on.
    """
    logging.info("Waiting for application {} on port {} to be active.".
      format(str(app), str(port)))
    if not wait_on_app(port):
      # In case the AppServer fails we let the AppController to detect it
      # and remove it if it still show in monit.
      logging.warning("AppServer did not come up in time, for {}:{}.".
        format(str(app), str(port)))
      return
  
    acc = appscale_info.get_appcontroller_client()
    appserver_ip = appscale_info.get_private_ip()
  
    while True:
      result = acc.add_routing_for_appserver(app, appserver_ip, port)
      if result == AppControllerClient.NOT_READY:
        logging.info('AppController not yet ready to add routing.')
        time.sleep(ROUTING_RETRY_INTERVAL)
      else:
        break
  
    logging.info('Successfully established routing for {} on port {}'.
      format(app, port))
  
  def start_app(config):
    """ Starts a Google App Engine application on this machine. It
        will start it up and then proceed to fetch the main page.
  
    Args:
      config: a dictionary that contains
         app_name: Name of the application to start
         app_port: Port to start on
         language: What language the app is written in
         login_ip: Public ip of deployment
         env_vars: A dict of environment variables that should be passed to the
          app.
         max_memory: An int that names the maximum amount of memory that this
          App Engine app is allowed to consume before being restarted.
         syslog_server: The IP of the syslog server to send the application
           logs to. Usually it's the login private IP.
    Returns:
      PID of process on success, -1 otherwise
    """
    config = convert_config_from_json(config)
    if config is None:
      logging.error("Invalid configuration for application")
      return BAD_PID
  
    if not misc.is_app_name_valid(config['app_name']):
      logging.error("Invalid app name for application: " + config['app_name'])
      return BAD_PID
    logging.info("Starting %s application %s" % (
      config['language'], config['app_name']))
  
    env_vars = config['env_vars']
    pidfile = PIDFILE_TEMPLATE.format(project=config['app_name'],
                                      port=config['app_port'])
  
    if config['language'] == constants.GO:
      env_vars['GOPATH'] = os.path.join('/var', 'apps', config['app_name'],
                                        'gopath')
      env_vars['GOROOT'] = os.path.join(GO_SDK, 'goroot')
  
    watch = "app___" + config['app_name']
    match_cmd = ""
  
    if config['language'] == constants.PYTHON27 or \
        config['language'] == constants.GO or \
        config['language'] == constants.PHP:
      start_cmd = create_python27_start_cmd(
        config['app_name'],
        config['login_ip'],
        config['app_port'],
        pidfile)
      stop_cmd = create_python27_stop_cmd(config['app_port'])
      env_vars.update(create_python_app_env(
        config['login_ip'],
        config['app_name']))
    elif config['language'] == constants.JAVA:
      remove_conflicting_jars(config['app_name'])
      copy_successful = copy_modified_jars(config['app_name'])
      if not copy_successful:
        return BAD_PID
  
      # Account for MaxPermSize (~170MB), the parent process (~50MB), and thread
      # stacks (~20MB).
      max_heap = config['max_memory'] - 250
      if max_heap <= 0:
        return BAD_PID
      start_cmd = create_java_start_cmd(
        config['app_name'],
        config['app_port'],
        config['login_ip'],
        max_heap,
        pidfile
      )
      match_cmd = "java -ea -cp.*--port={}.*{}".format(str(config['app_port']),
        os.path.dirname(locate_dir("/var/apps/" + config['app_name'] + "/app/",
        "WEB-INF")))
  
      stop_cmd = create_java_stop_cmd(config['app_port'])
      env_vars.update(create_java_app_env(config['app_name']))
    else:
      logging.error("Unknown application language %s for appname %s" \
        % (config['language'], config['app_name']))
      return BAD_PID
  
    logging.info("Start command: " + str(start_cmd))
    logging.info("Stop command: " + str(stop_cmd))
    logging.info("Environment variables: " + str(env_vars))
  
    # Set the syslog_server is specified.
    syslog_server = ""
    if 'syslog_server' in config:
      syslog_server = config['syslog_server']
    monit_app_configuration.create_config_file(
      watch,
      start_cmd,
      pidfile,
      config['app_port'],
      env_vars,
      config['max_memory'],
      syslog_server,
      check_port=True)
  
    # We want to tell monit to start the single process instead of the
    # group, since monit can get slow if there are quite a few processes in
    # the same group.
    full_watch = "{}-{}".format(str(watch), str(config['app_port']))
    if not monit_interface.start(full_watch, is_group=False):
      logging.warning("Monit was unable to start {}:{}".
        format(str(config['app_name']), config['app_port']))
      return BAD_PID
  
    # Since we are going to wait, possibly for a long time for the
    # application to be ready, we do it in a thread.
    threading.Thread(target=add_routing,
      args=(config['app_name'], config['app_port'])).start()
  
    if 'log_size' in config.keys():
      log_size = config['log_size']
    else:
      if config['app_name'] == APPSCALE_DASHBOARD_ID:
        log_size = DASHBOARD_LOG_SIZE
      else:
        log_size = APP_LOG_SIZE
  
    if not setup_logrotate(config['app_name'], watch, log_size):
      logging.error("Error while setting up log rotation for application: {}".
        format(config['app_name']))
  
    return 0
  
  def setup_logrotate(app_name, watch, log_size):
    """ Creates a logrotate script for the logs that the given application
        will create.
  
    Args:
      app_name: A string, the application ID.
      watch: A string of the form 'app___<app_ID>'.
      log_size: An integer, the size of logs that are kept per application server.
        The size should be in bytes.
    Returns:
      True on success, False otherwise.
    """
    # Write application specific logrotation script.
    app_logrotate_script = "{0}/appscale-{1}".\
      format(LOGROTATE_CONFIG_DIR, app_name)
  
    # Application logrotate script content.
    contents = """/var/log/appscale/{watch}*.log {{
    size {size}
    missingok
    rotate 7
    compress
    delaycompress
    notifempty
    copytruncate
  }}
  """.format(watch=watch, size=log_size)
    logging.debug("Logrotate file: {} - Contents:\n{}".
      format(app_logrotate_script, contents))
  
    with open(app_logrotate_script, 'w') as app_logrotate_fd:
      app_logrotate_fd.write(contents)
  
    return True
  
  def kill_instance(watch, instance_pid):
    """ Stops an AppServer process.
  
    Args:
      watch: A string specifying the monit entry for the process.
      instance_pid: An integer specifying the process ID.
    """
    process = psutil.Process(instance_pid)
    process.terminate()
    try:
      process.wait(MAX_INSTANCE_RESPONSE_TIME)
    except psutil.TimeoutExpired:
      process.kill()
  
    logging.info('Finished stopping {}'.format(watch))
  
  def unmonitor(process_name, retries=5):
    """ Unmonitors a process.
  
    Args:
      process_name: A string specifying the process to stop monitoring.
      retries: An integer specifying the number of times to retry the operation.
    """
    client = HTTPClient()
    process_url = '{}/{}'.format(monit_operator.LOCATION, process_name)
    payload = urllib.urlencode({'action': 'unmonitor'})
    try:
      client.fetch(process_url, method='POST', body=payload)
    except HTTPError as error:
      if error.code == 503:
        retries -= 1
        if retries < 0:
          raise
  
        return unmonitor(process_name, retries)
  
      raise
  
  def stop_app_instance(app_name, port):
    """ Stops a Google App Engine application process instance on current
        machine.
  
    Args:
      app_name: A string, the name of application to stop.
      port: The port the application is running on.
    Returns:
      True on success, False otherwise.
    """
    if not misc.is_app_name_valid(app_name):
      logging.error("Unable to kill app process %s on port %d because of " \
        "invalid name for application" % (app_name, int(port)))
      return False
  
    logging.info("Stopping application %s" % app_name)
    watch = "app___" + app_name + "-" + str(port)
  
    pid_location = os.path.join(constants.PID_DIR, '{}.pid'.format(watch))
    try:
      with open(pid_location) as pidfile:
        instance_pid = int(pidfile.read().strip())
    except IOError:
      logging.error('{} does not exist'.format(pid_location))
      return False
  
    unmonitor(watch)
  
    # Now that the AppServer is stopped, remove its monit config file so that
    # monit doesn't pick it up and restart it.
    monit_config_file = '{}/appscale-{}.cfg'.format(MONIT_CONFIG_DIR, watch)
    try:
      os.remove(monit_config_file)
    except OSError as os_error:
      logging.error("Error deleting {0}".format(monit_config_file))
  
    monit_interface.run_with_retry([monit_interface.MONIT, 'reload'])
    threading.Thread(target=kill_instance, args=(watch, instance_pid)).start()
    return True
  
  
  def stop_app(app_name):
    """ Stops all process instances of a Google App Engine application on this
        machine.
  
    Args:
      app_name: Name of application to stop
    Returns:
      True on success, False otherwise
    """
    if not misc.is_app_name_valid(app_name):
      logging.error("Unable to kill app process %s on because of " \
        "invalid name for application" % (app_name))
      return False
  
    logging.info("Stopping application %s" % app_name)
    watch = "app___" + app_name
    monit_result = monit_interface.stop(watch)
  
    if not monit_result:
      logging.error("Unable to shut down monit interface for watch %s" % watch)
      return False
  
    # Remove the monit config files for the application.
    # TODO: Reload monit to pick up config changes.
    config_files = glob.glob('{}/appscale-{}-*.cfg'.format(MONIT_CONFIG_DIR, watch))
    for config_file in config_files:
      try:
        os.remove(config_file)
      except OSError:
        logging.exception('Error removing {}'.format(config_file))
  
    if not remove_logrotate(app_name):
      logging.error("Error while setting up log rotation for application: {}".
        format(app_name))
  
    return True
  
  def remove_logrotate(app_name):
    """ Removes logrotate script for the given application.
  
    Args:
      app_name: A string, the name of the application to remove logrotate for.
    Returns:
      True on success, False otherwise.
    """
    app_logrotate_script = "{0}/appscale-{1}".\
      format(LOGROTATE_CONFIG_DIR, app_name)
    logging.debug("Removing script: {}".format(app_logrotate_script))
  
    try:
      os.remove(app_logrotate_script)
    except OSError:
      logging.error("Error deleting {0}".format(app_logrotate_script))
      return False
  
    return True
  
  ############################################
  # Private Functions (but public for testing)
  ############################################
  def wait_on_app(port):
    """ Waits for the application hosted on this machine, on the given port,
        to respond to HTTP requests.
  
    Args:
      port: Port where app is hosted on the local machine
    Returns:
      True on success, False otherwise
    """
    retries = math.ceil(START_APP_TIMEOUT / BACKOFF_TIME)
    private_ip = appscale_info.get_private_ip()
  
    url = "http://" + private_ip + ":" + str(port) + FETCH_PATH
    while retries > 0:
      try:
        opener = urllib2.build_opener(NoRedirection)
        response = opener.open(url)
        if response.code != HTTP_OK:
          logging.warning('{} returned {}. Headers: {}'.
            format(url, response.code, response.headers.headers))
        return True
      except IOError:
        retries -= 1
  
      time.sleep(BACKOFF_TIME)
  
    logging.error('Application did not come up on {} after {} seconds'.
      format(url, START_APP_TIMEOUT))
    return False
  
  def create_python_app_env(public_ip, app_name):
    """ Returns the environment variables the python application server uses.
  
    Args:
      public_ip: The public IP of the load balancer
      app_name: The name of the application to be run
    Returns:
      A dictionary containing the environment variables
    """
    env_vars = {}
    env_vars['MY_IP_ADDRESS'] = public_ip
    env_vars['APPNAME'] = app_name
    env_vars['GOMAXPROCS'] = appscale_info.get_num_cpus()
    env_vars['APPSCALE_HOME'] = constants.APPSCALE_HOME
    env_vars['PYTHON_LIB'] = "{0}/AppServer/".format(constants.APPSCALE_HOME)
    return env_vars
  
  def find_web_xml(app_name):
    """ Returns the location of a Java application's appengine-web.xml file.
  
    Args:
      app_name: A string containing the application ID.
    Returns:
      A string containing the location of the file.
    Raises:
      BadConfigurationException if the file is not found or multiple candidates
      are found.
  
    """
    app_dir = '/var/apps/{}/app'.format(app_name)
    file_name = 'appengine-web.xml'
    matches = []
    for root, dirs, files in os.walk(app_dir):
      if file_name in files and root.endswith('/WEB-INF'):
        matches.append(os.path.join(root, file_name))
  
    if len(matches) < 1:
      raise BadConfigurationException(
        'Unable to find {} file for {}'.format(file_name, app_name))
    if len(matches) > 1:
      # Use the shortest path. If there are any ties, use the first after
      # sorting alphabetically.
      matches.sort()
      match_to_use = matches[0]
      for match in matches:
        if len(match) < len(match_to_use):
          match_to_use = match
      return match_to_use
    return matches[0]
  
  def extract_env_vars_from_xml(xml_file):
    """ Returns any custom environment variables defined in appengine-web.xml.
  
    Args:
      xml_file: A string containing the location of the xml file.
    Returns:
      A dictionary containing the custom environment variables.
    """
    custom_vars = {}
    tree = ElementTree.parse(xml_file)
    root = tree.getroot()
    for child in root:
      if not child.tag.endswith('env-variables'):
        continue
  
      for env_var in child:
        var_dict = env_var.attrib
        custom_vars[var_dict['name']] = var_dict['value']
  
    return custom_vars
  
  def create_java_app_env(app_name):
    """ Returns the environment variables Java application servers uses.
  
    Args:
      app_name: A string containing the application ID.
    Returns:
      A dictionary containing the environment variables
    """
    env_vars = {'APPSCALE_HOME': constants.APPSCALE_HOME}
  
    config_file = find_web_xml(app_name)
    custom_env_vars = extract_env_vars_from_xml(config_file)
    env_vars.update(custom_env_vars)
  
    gcs_config = {'scheme': 'https', 'port': 443}
    try:
      gcs_config.update(deployment_config.get_config('gcs'))
    except ConfigInaccessible:
      logging.warning('Unable to fetch GCS configuration.')
  
    if 'host' in gcs_config:
      env_vars['GCS_HOST'] = '{scheme}://{host}:{port}'.format(**gcs_config)
  
    return env_vars
  
  def create_python27_start_cmd(app_name, login_ip, port, pidfile):
    """ Creates the start command to run the python application server.
  
    Args:
      app_name: The name of the application to run
      login_ip: The public IP of this deployment
      port: The local port the application server will bind to
      pidfile: A string specifying the pidfile location.
    Returns:
      A string of the start command.
    """
    db_proxy = appscale_info.get_db_proxy()
  
    cmd = [
      "/usr/bin/python2",
      constants.APPSCALE_HOME + "/AppServer/dev_appserver.py",
      "--port " + str(port),
      "--admin_port " + str(port + 10000),
      "--login_server " + login_ip,
      "--skip_sdk_update_check",
      "--nginx_host " + str(login_ip),
      "--require_indexes",
      "--enable_sendmail",
      "--xmpp_path " + login_ip,
      "--php_executable_path=" + str(PHP_CGI_LOCATION),
      "--uaserver_path " + db_proxy + ":"\
        + str(constants.UA_SERVER_PORT),
      "--datastore_path " + db_proxy + ":"\
        + str(constants.DB_SERVER_PORT),
      "/var/apps/" + app_name + "/app",
      "--host " + appscale_info.get_private_ip(),
      "--admin_host " + appscale_info.get_private_ip(),
      "--automatic_restart", "no",
      "--pidfile", pidfile]
  
    if app_name in TRUSTED_APPS:
      cmd.extend([TRUSTED_FLAG])
  
    return ' '.join(cmd)
  
  def locate_dir(path, dir_name):
    """ Locates a directory inside the given path.
  
    Args:
      path: The path to be searched
      dir_name: The directory we are looking for
  
    Returns:
      The absolute path of the directory we are looking for, None otherwise.
    """
    paths = []
  
    for root, sub_dirs, files in os.walk(path):
      for sub_dir in sub_dirs:
        if dir_name == sub_dir:
          result = os.path.abspath(os.path.join(root, sub_dir))
          if sub_dir == "WEB-INF":
            logging.info("Found WEB-INF/ at: {0}".format(result))
            paths.append(result)
          elif sub_dir == "lib" and result.count(os.sep) <= path.count(os.sep) + 2 \
              and result.endswith("/WEB-INF/{0}".format(sub_dir)):
            logging.info("Found lib/ at: {0}".format(result))
            paths.append(result)
  
    if len(paths) > 0:
      sorted_paths = sorted(paths, key = lambda s: len(s))
      return sorted_paths[0]
    else:
      return None
  
  def remove_conflicting_jars(app_name):
    """ Removes jars uploaded which may conflict with AppScale jars.
  
    Args:
      app_name: The name of the application to run.
    """
    app_dir = "/var/apps/" + app_name + "/app/"
    lib_dir = locate_dir(app_dir, "lib")
    if not lib_dir:
      logging.warn("Lib directory not found in app code while updating.")
      return
    logging.info("Removing jars from {0}".format(lib_dir))
    conflicting_jars_pattern = ['appengine-api-1.0-sdk-*.jar', 'appengine-api-stubs-*.jar',
                    'appengine-api-labs-*.jar', 'appengine-jsr107cache-*.jar',
                    'jsr107cache-*.jar', 'appengine-mapreduce*.jar',
                    'appengine-pipeline*.jar', 'appengine-gcs-client*.jar']
    for file in os.listdir(lib_dir):
      for pattern in conflicting_jars_pattern:
        if fnmatch.fnmatch(file, pattern):
          os.remove(lib_dir + os.sep + file)
  
  def copy_modified_jars(app_name):
    """ Copies the changes made to the Java SDK
    for AppScale into the apps lib folder.
  
    Args:
      app_name: The name of the application to run
  
    Returns:
      False if there were any errors, True if success
    """
    appscale_home = constants.APPSCALE_HOME
  
    app_dir = "/var/apps/" + app_name + "/app/"
    lib_dir = locate_dir(app_dir, "lib")
  
    if not lib_dir:
      web_inf_dir = locate_dir(app_dir, "WEB-INF")
      lib_dir = web_inf_dir + os.sep + "lib"
      logging.info("Creating lib directory at: {0}".format(lib_dir))
      mkdir_result = subprocess.call("mkdir " + lib_dir, shell=True)
  
      if mkdir_result != 0:
        logging.error("Failed to create missing lib directory in: {0}.".
          format(web_inf_dir))
        return False
    try:
      copy_files_matching_pattern(appscale_home + "/AppServer_Java/" +\
                  "appengine-java-sdk-repacked/lib/user/*.jar", lib_dir)
      copy_files_matching_pattern(appscale_home + "/AppServer_Java/" +\
                  "appengine-java-sdk-repacked/lib/impl/appscale-*.jar", lib_dir)
      copy_files_matching_pattern("/usr/share/appscale/ext/*", lib_dir)
    except IOError as io_error:
      logging.error("Failed to copy modified jar files to lib directory of " + app_name +\
                    " due to:" + str(io_error))
      return False
    return True
  
  def copy_files_matching_pattern(file_path_pattern, dest):
    """ Copies files matching the specified pattern to the destination directory.
    Args:
        file_path_pattern: The pattern of the files to be copied over.
        dest: The destination directory.
    """
    for file in glob.glob(file_path_pattern):
      shutil.copy(file, dest)
  
  def create_java_start_cmd(app_name, port, load_balancer_host, max_heap,
                            pidfile):
    """ Creates the start command to run the java application server.
  
    Args:
      app_name: The name of the application to run
      port: The local port the application server will bind to
      load_balancer_host: The host of the load balancer
      max_heap: An integer specifying the max heap size in MB.
      pidfile: A string specifying the pidfile location.
    Returns:
      A string of the start command.
    """
    db_proxy = appscale_info.get_db_proxy()
    tq_proxy = appscale_info.get_tq_proxy()
    java_start_script = os.path.join(
      constants.JAVA_APPSERVER, 'appengine-java-sdk-repacked', 'bin',
      'dev_appserver.sh')
  
    # The Java AppServer needs the NGINX_PORT flag set so that it will read the
    # local FS and see what port it's running on. The value doesn't matter.
    cmd = [
      java_start_script,
      "--port=" + str(port),
      #this jvm flag allows javax.email to connect to the smtp server
      "--jvm_flag=-Dsocket.permit_connect=true",
      '--jvm_flag=-Xmx{}m'.format(max_heap),
      '--jvm_flag=-Djava.security.egd=file:/dev/./urandom',
      "--disable_update_check",
      "--address=" + appscale_info.get_private_ip(),
      "--datastore_path=" + db_proxy,
      "--login_server=" + load_balancer_host,
      "--appscale_version=1",
      "--APP_NAME=" + app_name,
      "--NGINX_ADDRESS=" + load_balancer_host,
      "--TQ_PROXY=" + tq_proxy,
      "--pidfile={}".format(pidfile),
      os.path.dirname(locate_dir("/var/apps/" + app_name + "/app/", "WEB-INF"))
    ]
  
    return ' '.join(cmd)
  
  def create_python27_stop_cmd(port):
    """ This creates the stop command for an application which is
    uniquely identified by a port number. Additional portions of the
    start command are included to prevent the termination of other
    processes.
  
    Args:
      port: The port which the application server is running
    Returns:
      A string of the stop command.
    """
    stop_cmd = "/usr/bin/python2 {0}/scripts/stop_service.py " \
      "dev_appserver.py {1}".format(constants.APPSCALE_HOME, port)
    return stop_cmd
  
  def create_java_stop_cmd(port):
    """ This creates the stop command for an application which is
    uniquely identified by a port number. Additional portions of the
    start command are included to prevent the termination of other
    processes.
  
    Args:
      port: The port which the application server is running
    Returns:
      A string of the stop command.
    """
    stop_cmd = "/usr/bin/python2 {0}/scripts/stop_service.py " \
      "java {1}".format(constants.APPSCALE_HOME, port)
    return stop_cmd
  
  def is_config_valid(config):
    """ Takes a configuration and checks to make sure all required properties
      are present.
  
    Args:
      config: The dictionary to validate
    Returns:
      True if valid, False otherwise
    """
    for ii in REQUIRED_CONFIG_FIELDS:
      try:
        if config[ii]:
          pass
      except KeyError:
        logging.error("Unable to find " + str(ii) + " in configuration")
        return False
    return True
  
  
  ################################
  # MAIN
  ################################
  if __name__ == "__main__":
    file_io.set_logging_format()
  
    zk_ips = appscale_info.get_zk_node_ips()
    zk_client = KazooClient(hosts=','.join(zk_ips))
    zk_client.start()
    deployment_config = DeploymentConfig(zk_client)
  
    INTERNAL_IP = appscale_info.get_private_ip()
    SERVER = SOAPpy.SOAPServer((INTERNAL_IP, constants.APP_MANAGER_PORT))
  
    SERVER.registerFunction(start_app)
    SERVER.registerFunction(stop_app)
    SERVER.registerFunction(stop_app_instance)
  
    while 1:
      try:
        SERVER.serve_forever()
      except SSL.SSLError:
        pass
  
files: AppManager/app_manager_server.py
files: """ This service starts and stops application servers of a given application. """
  
  import fnmatch
  import glob
  import json
  import logging
  import math
  import os
  import psutil
  import shutil
  import SOAPpy
  import subprocess
  import sys
  import threading
  import time
  import urllib
  import urllib2
  from xml.etree import ElementTree
  
  from M2Crypto import SSL
  from tornado.httpclient import HTTPClient
  from tornado.httpclient import HTTPError
  
  from appscale.common import (
    appscale_info,
    constants,
    file_io,
    monit_app_configuration,
    monit_interface,
    misc
  )
  from appscale.common.deployment_config import DeploymentConfig
  from appscale.common.deployment_config import ConfigInaccessible
  from appscale.common.monit_app_configuration import MONIT_CONFIG_DIR
  from appscale.common.monit_interface import MonitOperator
  from appscale.common.unpackaged import APPSCALE_PYTHON_APPSERVER
  
  sys.path.append(APPSCALE_PYTHON_APPSERVER)
  from google.appengine.api.appcontroller_client import AppControllerClient
  
  # The amount of seconds to wait for an application to start up.
  START_APP_TIMEOUT = 180
  
  # The amount of seconds to wait between checking if an application is up.
  BACKOFF_TIME = 1
  
  # The PID number to return when a process did not start correctly
  BAD_PID = -1
  
  # Default hourly cron directory.
  CRON_HOURLY = '/etc/cron.hourly'
  
  # Default logrotate configuration directory.
  LOGROTATE_CONFIG_DIR = '/etc/logrotate.d'
  
  # Max log size for AppScale Dashboard servers.
  DASHBOARD_LOG_SIZE = 10 * 1024 * 1024
  
  # Max application server log size in bytes.
  APP_LOG_SIZE = 250 * 1024 * 1024
  
  # Required configuration fields for starting an application
  REQUIRED_CONFIG_FIELDS = [
    'app_name',
    'app_port',
    'language',
    'login_ip',
    'env_vars',
    'max_memory']
  
  # The web path to fetch to see if the application is up
  FETCH_PATH = '/_ah/health_check'
  
  # The app ID of the AppScale Dashboard.
  APPSCALE_DASHBOARD_ID = "appscaledashboard"
  
  # Apps which can access any application's data.
  TRUSTED_APPS = ["appscaledashboard"]
  
  # The flag to tell the application server that this application can access
  # all application data.
  TRUSTED_FLAG = "--trusted"
  
  # The location on the filesystem where the PHP executable is installed.
  PHP_CGI_LOCATION = "/usr/bin/php-cgi"
  
  # The location of the App Engine SDK for Go.
  GO_SDK = os.path.join('/', 'opt', 'go_appengine')
  
  HTTP_OK = 200
  
  # The amount of seconds to wait before retrying to add routing.
  ROUTING_RETRY_INTERVAL = 5
  
  PIDFILE_TEMPLATE = os.path.join('/', 'var', 'run', 'appscale',
                                  'app___{project}-{port}.pid')
  
  # The number of seconds an instance is allowed to finish serving requests after
  # it receives a shutdown signal.
  MAX_INSTANCE_RESPONSE_TIME = 600
  
  # A DeploymentConfig accessor.
  deployment_config = None
  
  # An interface for working with Monit.
  monit_operator = MonitOperator()
  
  
  class BadConfigurationException(Exception):
    """ An application is configured incorrectly. """
    def __init__(self, value):
      Exception.__init__(self, value)
      self.value = value
  
    def __str__(self):
      return repr(self.value)
  
  class NoRedirection(urllib2.HTTPErrorProcessor):
    """ A url opener that does not automatically redirect. """
    def http_response(self, request, response):
      """ Processes HTTP responses.
  
      Args:
        request: An HTTP request object.
        response: An HTTP response object.
      Returns:
        The HTTP response object.
      """
      return response
    https_response = http_response
  
  def convert_config_from_json(config):
    """ Takes the configuration in JSON format and converts it to a dictionary.
        Validates the dictionary configuration before returning.
  
    Args:
      config: The configuration to convert
    Returns:
      None if it failed to convert the config and a dictionary if it succeeded
    """
    logging.info("Configuration for app:" + str(config))
    try:
      config = json.loads(config)
    except ValueError, error:
      logging.error("%s Exception--Unable to parse configuration: %s" % \
        (error.__class__, str(error)))
      return None
    except TypeError, error:
      logging.error("%s Exception--Unable to parse configuration: %s" % \
        (error.__class__, str(error)))
      return None
  
    if is_config_valid(config):
      return config
    else:
      return None
  
  def add_routing(app, port):
    """ Tells the AppController to begin routing traffic to an AppServer.
  
    Args:
      app: A string that contains the application ID.
      port: A string that contains the port that the AppServer listens on.
    """
    logging.info("Waiting for application {} on port {} to be active.".
      format(str(app), str(port)))
    if not wait_on_app(port):
      # In case the AppServer fails we let the AppController to detect it
      # and remove it if it still show in monit.
      logging.warning("AppServer did not come up in time, for {}:{}.".
        format(str(app), str(port)))
      return
  
    acc = appscale_info.get_appcontroller_client()
    appserver_ip = appscale_info.get_private_ip()
  
    while True:
      result = acc.add_routing_for_appserver(app, appserver_ip, port)
      if result == AppControllerClient.NOT_READY:
        logging.info('AppController not yet ready to add routing.')
        time.sleep(ROUTING_RETRY_INTERVAL)
      else:
        break
  
    logging.info('Successfully established routing for {} on port {}'.
      format(app, port))
  
  def start_app(config):
    """ Starts a Google App Engine application on this machine. It
        will start it up and then proceed to fetch the main page.
  
    Args:
      config: a dictionary that contains
         app_name: Name of the application to start
         app_port: Port to start on
         language: What language the app is written in
         login_ip: Public ip of deployment
         env_vars: A dict of environment variables that should be passed to the
          app.
         max_memory: An int that names the maximum amount of memory that this
          App Engine app is allowed to consume before being restarted.
         syslog_server: The IP of the syslog server to send the application
           logs to. Usually it's the login private IP.
    Returns:
      PID of process on success, -1 otherwise
    """
    config = convert_config_from_json(config)
    if config is None:
      logging.error("Invalid configuration for application")
      return BAD_PID
  
    if not misc.is_app_name_valid(config['app_name']):
      logging.error("Invalid app name for application: " + config['app_name'])
      return BAD_PID
    logging.info("Starting %s application %s" % (
      config['language'], config['app_name']))
  
    env_vars = config['env_vars']
    pidfile = PIDFILE_TEMPLATE.format(project=config['app_name'],
                                      port=config['app_port'])
  
    if config['language'] == constants.GO:
      env_vars['GOPATH'] = os.path.join('/var', 'apps', config['app_name'],
                                        'gopath')
      env_vars['GOROOT'] = os.path.join(GO_SDK, 'goroot')
  
    watch = "app___" + config['app_name']
    match_cmd = ""
  
    if config['language'] == constants.PYTHON27 or \
        config['language'] == constants.GO or \
        config['language'] == constants.PHP:
      start_cmd = create_python27_start_cmd(
        config['app_name'],
        config['login_ip'],
        config['app_port'],
        pidfile)
      stop_cmd = create_python27_stop_cmd(config['app_port'])
      env_vars.update(create_python_app_env(
        config['login_ip'],
        config['app_name']))
    elif config['language'] == constants.JAVA:
      remove_conflicting_jars(config['app_name'])
      copy_successful = copy_modified_jars(config['app_name'])
      if not copy_successful:
        return BAD_PID
  
      # Account for MaxPermSize (~170MB), the parent process (~50MB), and thread
      # stacks (~20MB).
      max_heap = config['max_memory'] - 250
      if max_heap <= 0:
        return BAD_PID
      start_cmd = create_java_start_cmd(
        config['app_name'],
        config['app_port'],
        config['login_ip'],
        max_heap,
        pidfile
      )
      match_cmd = "java -ea -cp.*--port={}.*{}".format(str(config['app_port']),
        os.path.dirname(locate_dir("/var/apps/" + config['app_name'] + "/app/",
        "WEB-INF")))
  
      stop_cmd = create_java_stop_cmd(config['app_port'])
      env_vars.update(create_java_app_env(config['app_name']))
    else:
      logging.error("Unknown application language %s for appname %s" \
        % (config['language'], config['app_name']))
      return BAD_PID
  
    logging.info("Start command: " + str(start_cmd))
    logging.info("Stop command: " + str(stop_cmd))
    logging.info("Environment variables: " + str(env_vars))
  
    # Set the syslog_server is specified.
    syslog_server = ""
    if 'syslog_server' in config:
      syslog_server = config['syslog_server']
    monit_app_configuration.create_config_file(
      watch,
      start_cmd,
      pidfile,
      config['app_port'],
      env_vars,
      config['max_memory'],
      syslog_server,
      check_port=True)
  
    # We want to tell monit to start the single process instead of the
    # group, since monit can get slow if there are quite a few processes in
    # the same group.
    full_watch = "{}-{}".format(str(watch), str(config['app_port']))
    if not monit_interface.start(full_watch, is_group=False):
      logging.warning("Monit was unable to start {}:{}".
        format(str(config['app_name']), config['app_port']))
      return BAD_PID
  
    # Since we are going to wait, possibly for a long time for the
    # application to be ready, we do it in a thread.
    threading.Thread(target=add_routing,
      args=(config['app_name'], config['app_port'])).start()
  
    if 'log_size' in config.keys():
      log_size = config['log_size']
    else:
      if config['app_name'] == APPSCALE_DASHBOARD_ID:
        log_size = DASHBOARD_LOG_SIZE
      else:
        log_size = APP_LOG_SIZE
  
    if not setup_logrotate(config['app_name'], watch, log_size):
      logging.error("Error while setting up log rotation for application: {}".
        format(config['app_name']))
  
    return 0
  
  def setup_logrotate(app_name, watch, log_size):
    """ Creates a logrotate script for the logs that the given application
        will create.
  
    Args:
      app_name: A string, the application ID.
      watch: A string of the form 'app___<app_ID>'.
      log_size: An integer, the size of logs that are kept per application server.
        The size should be in bytes.
    Returns:
      True on success, False otherwise.
    """
    # Write application specific logrotation script.
    app_logrotate_script = "{0}/appscale-{1}".\
      format(LOGROTATE_CONFIG_DIR, app_name)
  
    # Application logrotate script content.
    contents = """/var/log/appscale/{watch}*.log {{
    size {size}
    missingok
    rotate 7
    compress
    delaycompress
    notifempty
    copytruncate
  }}
  """.format(watch=watch, size=log_size)
    logging.debug("Logrotate file: {} - Contents:\n{}".
      format(app_logrotate_script, contents))
  
    with open(app_logrotate_script, 'w') as app_logrotate_fd:
      app_logrotate_fd.write(contents)
  
    return True
  
  def kill_instance(watch, instance_pid):
    """ Stops an AppServer process.
  
    Args:
      watch: A string specifying the monit entry for the process.
      instance_pid: An integer specifying the process ID.
    """
    process = psutil.Process(instance_pid)
    process.terminate()
    try:
      process.wait(MAX_INSTANCE_RESPONSE_TIME)
    except psutil.TimeoutExpired:
      process.kill()
  
    logging.info('Finished stopping {}'.format(watch))
  
  def unmonitor(process_name, retries=5):
    """ Unmonitors a process.
  
    Args:
      process_name: A string specifying the process to stop monitoring.
      retries: An integer specifying the number of times to retry the operation.
    """
    client = HTTPClient()
    process_url = '{}/{}'.format(monit_operator.LOCATION, process_name)
    payload = urllib.urlencode({'action': 'unmonitor'})
    try:
      client.fetch(process_url, method='POST', body=payload)
    except HTTPError as error:
      if error.code == 503:
        retries -= 1
        if retries < 0:
          raise
  
        return unmonitor(process_name, retries)
  
      raise
  
  def stop_app_instance(app_name, port):
    """ Stops a Google App Engine application process instance on current
        machine.
  
    Args:
      app_name: A string, the name of application to stop.
      port: The port the application is running on.
    Returns:
      True on success, False otherwise.
    """
    if not misc.is_app_name_valid(app_name):
      logging.error("Unable to kill app process %s on port %d because of " \
        "invalid name for application" % (app_name, int(port)))
      return False
  
    logging.info("Stopping application %s" % app_name)
    watch = "app___" + app_name + "-" + str(port)
  
    pid_location = os.path.join(constants.PID_DIR, '{}.pid'.format(watch))
    try:
      with open(pid_location) as pidfile:
        instance_pid = int(pidfile.read().strip())
    except IOError:
      logging.error('{} does not exist'.format(pid_location))
      return False
  
    unmonitor(watch)
  
    # Now that the AppServer is stopped, remove its monit config file so that
    # monit doesn't pick it up and restart it.
    monit_config_file = '{}/appscale-{}.cfg'.format(MONIT_CONFIG_DIR, watch)
    try:
      os.remove(monit_config_file)
    except OSError as os_error:
      logging.error("Error deleting {0}".format(monit_config_file))
  
    monit_interface.run_with_retry([monit_interface.MONIT, 'reload'])
    threading.Thread(target=kill_instance, args=(watch, instance_pid)).start()
    return True
  
  
  def stop_app(app_name):
    """ Stops all process instances of a Google App Engine application on this
        machine.
  
    Args:
      app_name: Name of application to stop
    Returns:
      True on success, False otherwise
    """
    if not misc.is_app_name_valid(app_name):
      logging.error("Unable to kill app process %s on because of " \
        "invalid name for application" % (app_name))
      return False
  
    logging.info("Stopping application %s" % app_name)
    watch = "app___" + app_name
    monit_result = monit_interface.stop(watch)
  
    if not monit_result:
      logging.error("Unable to shut down monit interface for watch %s" % watch)
      return False
  
    # Remove the monit config files for the application.
    # TODO: Reload monit to pick up config changes.
    config_files = glob.glob('{}/appscale-{}-*.cfg'.format(MONIT_CONFIG_DIR, watch))
    for config_file in config_files:
      try:
        os.remove(config_file)
      except OSError:
        logging.exception('Error removing {}'.format(config_file))
  
    if not remove_logrotate(app_name):
      logging.error("Error while setting up log rotation for application: {}".
        format(app_name))
  
    return True
  
  def remove_logrotate(app_name):
    """ Removes logrotate script for the given application.
  
    Args:
      app_name: A string, the name of the application to remove logrotate for.
    Returns:
      True on success, False otherwise.
    """
    app_logrotate_script = "{0}/appscale-{1}".\
      format(LOGROTATE_CONFIG_DIR, app_name)
    logging.debug("Removing script: {}".format(app_logrotate_script))
  
    try:
      os.remove(app_logrotate_script)
    except OSError:
      logging.error("Error deleting {0}".format(app_logrotate_script))
      return False
  
    return True
  
  ############################################
  # Private Functions (but public for testing)
  ############################################
  def wait_on_app(port):
    """ Waits for the application hosted on this machine, on the given port,
        to respond to HTTP requests.
  
    Args:
      port: Port where app is hosted on the local machine
    Returns:
      True on success, False otherwise
    """
    retries = math.ceil(START_APP_TIMEOUT / BACKOFF_TIME)
    private_ip = appscale_info.get_private_ip()
  
    url = "http://" + private_ip + ":" + str(port) + FETCH_PATH
    while retries > 0:
      try:
        opener = urllib2.build_opener(NoRedirection)
        response = opener.open(url)
        if response.code != HTTP_OK:
          logging.warning('{} returned {}. Headers: {}'.
            format(url, response.code, response.headers.headers))
        return True
      except IOError:
        retries -= 1
  
      time.sleep(BACKOFF_TIME)
  
    logging.error('Application did not come up on {} after {} seconds'.
      format(url, START_APP_TIMEOUT))
    return False
  
  def create_python_app_env(public_ip, app_name):
    """ Returns the environment variables the python application server uses.
  
    Args:
      public_ip: The public IP of the load balancer
      app_name: The name of the application to be run
    Returns:
      A dictionary containing the environment variables
    """
    env_vars = {}
    env_vars['MY_IP_ADDRESS'] = public_ip
    env_vars['APPNAME'] = app_name
    env_vars['GOMAXPROCS'] = appscale_info.get_num_cpus()
    env_vars['APPSCALE_HOME'] = constants.APPSCALE_HOME
    env_vars['PYTHON_LIB'] = "{0}/AppServer/".format(constants.APPSCALE_HOME)
    return env_vars
  
  def find_web_xml(app_name):
    """ Returns the location of a Java application's appengine-web.xml file.
  
    Args:
      app_name: A string containing the application ID.
    Returns:
      A string containing the location of the file.
    Raises:
      BadConfigurationException if the file is not found or multiple candidates
      are found.
  
    """
    app_dir = '/var/apps/{}/app'.format(app_name)
    file_name = 'appengine-web.xml'
    matches = []
    for root, dirs, files in os.walk(app_dir):
      if file_name in files and root.endswith('/WEB-INF'):
        matches.append(os.path.join(root, file_name))
  
    if len(matches) < 1:
      raise BadConfigurationException(
        'Unable to find {} file for {}'.format(file_name, app_name))
    if len(matches) > 1:
      # Use the shortest path. If there are any ties, use the first after
      # sorting alphabetically.
      matches.sort()
      match_to_use = matches[0]
      for match in matches:
        if len(match) < len(match_to_use):
          match_to_use = match
      return match_to_use
    return matches[0]
  
  def extract_env_vars_from_xml(xml_file):
    """ Returns any custom environment variables defined in appengine-web.xml.
  
    Args:
      xml_file: A string containing the location of the xml file.
    Returns:
      A dictionary containing the custom environment variables.
    """
    custom_vars = {}
    tree = ElementTree.parse(xml_file)
    root = tree.getroot()
    for child in root:
      if not child.tag.endswith('env-variables'):
        continue
  
      for env_var in child:
        var_dict = env_var.attrib
        custom_vars[var_dict['name']] = var_dict['value']
  
    return custom_vars
  
  def create_java_app_env(app_name):
    """ Returns the environment variables Java application servers uses.
  
    Args:
      app_name: A string containing the application ID.
    Returns:
      A dictionary containing the environment variables
    """
    env_vars = {'APPSCALE_HOME': constants.APPSCALE_HOME}
  
    config_file = find_web_xml(app_name)
    custom_env_vars = extract_env_vars_from_xml(config_file)
    env_vars.update(custom_env_vars)
  
    gcs_config = {'scheme': 'https', 'port': 443}
    try:
      gcs_config.update(deployment_config.get_config('gcs'))
    except ConfigInaccessible:
      logging.warning('Unable to fetch GCS configuration.')
  
    if 'host' in gcs_config:
      env_vars['GCS_HOST'] = '{scheme}://{host}:{port}'.format(**gcs_config)
  
    return env_vars
  
  def create_python27_start_cmd(app_name, login_ip, port, pidfile):
    """ Creates the start command to run the python application server.
  
    Args:
      app_name: The name of the application to run
      login_ip: The public IP of this deployment
      port: The local port the application server will bind to
      pidfile: A string specifying the pidfile location.
    Returns:
      A string of the start command.
    """
    db_proxy = appscale_info.get_db_proxy()
  
    cmd = [
      "/usr/bin/python2",
      constants.APPSCALE_HOME + "/AppServer/dev_appserver.py",
      "--port " + str(port),
      "--admin_port " + str(port + 10000),
      "--login_server " + login_ip,
      "--skip_sdk_update_check",
      "--nginx_host " + str(login_ip),
      "--require_indexes",
      "--enable_sendmail",
      "--xmpp_path " + login_ip,
      "--php_executable_path=" + str(PHP_CGI_LOCATION),
      "--uaserver_path " + db_proxy + ":"\
        + str(constants.UA_SERVER_PORT),
      "--datastore_path " + db_proxy + ":"\
        + str(constants.DB_SERVER_PORT),
      "/var/apps/" + app_name + "/app",
      "--host " + appscale_info.get_private_ip(),
      "--admin_host " + appscale_info.get_private_ip(),
      "--automatic_restart", "no",
      "--pidfile", pidfile]
  
    if app_name in TRUSTED_APPS:
      cmd.extend([TRUSTED_FLAG])
  
    return ' '.join(cmd)
  
  def locate_dir(path, dir_name):
    """ Locates a directory inside the given path.
  
    Args:
      path: The path to be searched
      dir_name: The directory we are looking for
  
    Returns:
      The absolute path of the directory we are looking for, None otherwise.
    """
    paths = []
  
    for root, sub_dirs, files in os.walk(path):
      for sub_dir in sub_dirs:
        if dir_name == sub_dir:
          result = os.path.abspath(os.path.join(root, sub_dir))
          if sub_dir == "WEB-INF":
            logging.info("Found WEB-INF/ at: {0}".format(result))
            paths.append(result)
          elif sub_dir == "lib" and result.count(os.sep) <= path.count(os.sep) + 2 \
              and result.endswith("/WEB-INF/{0}".format(sub_dir)):
            logging.info("Found lib/ at: {0}".format(result))
            paths.append(result)
  
    if len(paths) > 0:
      sorted_paths = sorted(paths, key = lambda s: len(s))
      return sorted_paths[0]
    else:
      return None
  
  def remove_conflicting_jars(app_name):
    """ Removes jars uploaded which may conflict with AppScale jars.
  
    Args:
      app_name: The name of the application to run.
    """
    app_dir = "/var/apps/" + app_name + "/app/"
    lib_dir = locate_dir(app_dir, "lib")
    if not lib_dir:
      logging.warn("Lib directory not found in app code while updating.")
      return
    logging.info("Removing jars from {0}".format(lib_dir))
    conflicting_jars_pattern = ['appengine-api-1.0-sdk-*.jar', 'appengine-api-stubs-*.jar',
                    'appengine-api-labs-*.jar', 'appengine-jsr107cache-*.jar',
                    'jsr107cache-*.jar', 'appengine-mapreduce*.jar',
                    'appengine-pipeline*.jar', 'appengine-gcs-client*.jar']
    for file in os.listdir(lib_dir):
      for pattern in conflicting_jars_pattern:
        if fnmatch.fnmatch(file, pattern):
          os.remove(lib_dir + os.sep + file)
  
  def copy_modified_jars(app_name):
    """ Copies the changes made to the Java SDK
    for AppScale into the apps lib folder.
  
    Args:
      app_name: The name of the application to run
  
    Returns:
      False if there were any errors, True if success
    """
    appscale_home = constants.APPSCALE_HOME
  
    app_dir = "/var/apps/" + app_name + "/app/"
    lib_dir = locate_dir(app_dir, "lib")
  
    if not lib_dir:
      web_inf_dir = locate_dir(app_dir, "WEB-INF")
      lib_dir = web_inf_dir + os.sep + "lib"
      logging.info("Creating lib directory at: {0}".format(lib_dir))
      mkdir_result = subprocess.call("mkdir " + lib_dir, shell=True)
  
      if mkdir_result != 0:
        logging.error("Failed to create missing lib directory in: {0}.".
          format(web_inf_dir))
        return False
    try:
      copy_files_matching_pattern(appscale_home + "/AppServer_Java/" +\
                  "appengine-java-sdk-repacked/lib/user/*.jar", lib_dir)
      copy_files_matching_pattern(appscale_home + "/AppServer_Java/" +\
                  "appengine-java-sdk-repacked/lib/impl/appscale-*.jar", lib_dir)
      copy_files_matching_pattern("/usr/share/appscale/ext/*", lib_dir)
    except IOError as io_error:
      logging.error("Failed to copy modified jar files to lib directory of " + app_name +\
                    " due to:" + str(io_error))
      return False
    return True
  
  def copy_files_matching_pattern(file_path_pattern, dest):
    """ Copies files matching the specified pattern to the destination directory.
    Args:
        file_path_pattern: The pattern of the files to be copied over.
        dest: The destination directory.
    """
    for file in glob.glob(file_path_pattern):
      shutil.copy(file, dest)
  
  def create_java_start_cmd(app_name, port, load_balancer_host, max_heap,
                            pidfile):
    """ Creates the start command to run the java application server.
  
    Args:
      app_name: The name of the application to run
      port: The local port the application server will bind to
      load_balancer_host: The host of the load balancer
      max_heap: An integer specifying the max heap size in MB.
      pidfile: A string specifying the pidfile location.
    Returns:
      A string of the start command.
    """
    db_proxy = appscale_info.get_db_proxy()
    tq_proxy = appscale_info.get_tq_proxy()
    java_start_script = os.path.join(
      constants.JAVA_APPSERVER, 'appengine-java-sdk-repacked', 'bin',
      'dev_appserver.sh')
  
    # The Java AppServer needs the NGINX_PORT flag set so that it will read the
    # local FS and see what port it's running on. The value doesn't matter.
    cmd = [
      java_start_script,
      "--port=" + str(port),
      #this jvm flag allows javax.email to connect to the smtp server
      "--jvm_flag=-Dsocket.permit_connect=true",
      '--jvm_flag=-Xmx{}m'.format(max_heap),
      '--jvm_flag=-Djava.security.egd=file:/dev/./urandom',
      "--disable_update_check",
      "--address=" + appscale_info.get_private_ip(),
      "--datastore_path=" + db_proxy,
      "--login_server=" + load_balancer_host,
      "--appscale_version=1",
      "--APP_NAME=" + app_name,
      "--NGINX_ADDRESS=" + load_balancer_host,
      "--TQ_PROXY=" + tq_proxy,
      "--pidfile={}".format(pidfile),
      os.path.dirname(locate_dir("/var/apps/" + app_name + "/app/", "WEB-INF"))
    ]
  
    return ' '.join(cmd)
  
  def create_python27_stop_cmd(port):
    """ This creates the stop command for an application which is
    uniquely identified by a port number. Additional portions of the
    start command are included to prevent the termination of other
    processes.
  
    Args:
      port: The port which the application server is running
    Returns:
      A string of the stop command.
    """
    stop_cmd = "/usr/bin/python2 {0}/scripts/stop_service.py " \
      "dev_appserver.py {1}".format(constants.APPSCALE_HOME, port)
    return stop_cmd
  
  def create_java_stop_cmd(port):
    """ This creates the stop command for an application which is
    uniquely identified by a port number. Additional portions of the
    start command are included to prevent the termination of other
    processes.
  
    Args:
      port: The port which the application server is running
    Returns:
      A string of the stop command.
    """
    stop_cmd = "/usr/bin/python2 {0}/scripts/stop_service.py " \
      "java {1}".format(constants.APPSCALE_HOME, port)
    return stop_cmd
  
  def is_config_valid(config):
    """ Takes a configuration and checks to make sure all required properties
      are present.
  
    Args:
      config: The dictionary to validate
    Returns:
      True if valid, False otherwise
    """
    for ii in REQUIRED_CONFIG_FIELDS:
      try:
        if config[ii]:
          pass
      except KeyError:
        logging.error("Unable to find " + str(ii) + " in configuration")
        return False
    return True
  
  
  ################################
  # MAIN
  ################################
  if __name__ == "__main__":
    file_io.set_logging_format()
    deployment_config = DeploymentConfig(appscale_info.get_zk_locations_string())
  
    INTERNAL_IP = appscale_info.get_private_ip()
    SERVER = SOAPpy.SOAPServer((INTERNAL_IP, constants.APP_MANAGER_PORT))
  
    SERVER.registerFunction(start_app)
    SERVER.registerFunction(stop_app)
    SERVER.registerFunction(stop_app_instance)
  
    while 1:
      try:
        SERVER.serve_forever()
      except SSL.SSLError:
        pass
  
files: AppManager/app_manager_server.py
files: import json
  import logging
  import time
  
  from kazoo.client import KazooException
  from kazoo.client import KazooState
  from kazoo.client import NoNodeError
  from kazoo.exceptions import ZookeeperError
  from threading import Lock
  
  from .constants import SMALL_WAIT
  from .constants import TINY_WAIT
  
  
  class ConfigInaccessible(Exception):
    """ Indicates that the configuration storage is not accessible. """
    pass
  
  
  class InvalidConfig(Exception):
    """ Indicates that a configuration option is invalid. """
    pass
  
  
  class ConfigStates(object):
    """ Possible states for DeploymentConfig. """
    LOADING = 'loading'
    LOADED = 'loaded'
  
  
  class DeploymentConfig(object):
    """ Accesses deployment configuration options. """
    # The ZooKeeper node where configuration is stored.
    CONFIG_ROOT = '/appscale/config'
  
    def __init__(self, zk_client):
      """ Creates new DeploymentConfig object.
  
      Args:
        zk_client: A KazooClient.
      """
      self.logger = logging.getLogger(self.__class__.__name__)
      self.update_lock = Lock()
      self.state = ConfigStates.LOADING
      self.config = {}
      self.conn = zk_client
      self.conn.add_listener(self._conn_listener)
      self.conn.ensure_path(self.CONFIG_ROOT)
      self.conn.ChildrenWatch(self.CONFIG_ROOT, func=self._update_config)
  
    def _conn_listener(self, state):
      """ Handles changes in ZooKeeper connection state.
  
      Args:
        state: A string indicating the new state.
      """
      if state == KazooState.LOST:
        self.logger.warning('ZK connection lost')
      if state == KazooState.SUSPENDED:
        self.logger.warning('ZK connection suspended')
      else:
        self.logger.info('ZK connection established')
  
    def _load_child(self, child):
      """ Fetches the data for a configuration node.
  
      Args:
        child: A string containing the ZooKeeper node to fetch.
      Returns:
        A dictionary containing configuration data.
      Raises:
        InaccessibleConfig if ZooKeeper is not accessible.
      """
      node = '/'.join([self.CONFIG_ROOT, child])
      try:
        data, _ = self.conn.retry(self.conn.get, node)
      except (KazooException, ZookeeperError):
        raise ConfigInaccessible('ZooKeeper connection not available')
      except NoNodeError:
        return {}
  
      try:
        return json.loads(data)
      except ValueError:
        self.logger.warning('Invalid deployment config: {}'.format(child))
        return {}
  
    def _update_config(self, children):
      """ Updates configuration when it changes.
  
      Args:
        children: A list of ZooKeeper nodes.
      """
      with self.update_lock:
        self.state = ConfigStates.LOADING
  
        # Ensure old sections are removed.
        self.config = {}
  
        for child in children:
          while True:
            try:
              self.config[child] = self._load_child(child)
              break
            except ConfigInaccessible as load_error:
              self.logger.warning(str(load_error))
              time.sleep(SMALL_WAIT)
  
        self.logger.info('Deployment configuration updated')
        self.state = ConfigStates.LOADED
  
    def get_config(self, section):
      """ Fetches the configuration for a given section.
  
      Args:
        section: A string specifying the section to fetch.
      Returns:
        A dictionary containing configuration data.
      Raises:
        InaccessibleConfig if ZooKeeper is inaccessible.
      """
      # If the connection is established, it should finish loading very soon.
      while (self.state == ConfigStates.LOADING and
             self.conn.state not in (KazooState.LOST, KazooState.SUSPENDED)):
        time.sleep(TINY_WAIT)
  
      if self.state != ConfigStates.LOADED:
        raise ConfigInaccessible('ZooKeeper connection not available')
  
      with self.update_lock:
        if section not in self.config:
          return {}
        return self.config[section]
  
    def close(self):
      """ Close the ZooKeeper connection. """
      self.conn.stop()
  
files: common/appscale/common/deployment_config.py
files: import json
  import logging
  import time
  
  from kazoo.client import KazooClient
  from kazoo.client import KazooException
  from kazoo.client import KazooState
  from kazoo.client import NoNodeError
  from kazoo.exceptions import ZookeeperError
  from threading import Lock
  
  from .constants import SMALL_WAIT
  from .constants import TINY_WAIT
  
  
  class ConfigInaccessible(Exception):
    """ Indicates that the configuration storage is not accessible. """
    pass
  
  
  class InvalidConfig(Exception):
    """ Indicates that a configuration option is invalid. """
    pass
  
  
  class ConfigStates(object):
    """ Possible states for DeploymentConfig. """
    LOADING = 'loading'
    LOADED = 'loaded'
  
  
  class DeploymentConfig(object):
    """ Accesses deployment configuration options. """
    # The ZooKeeper node where configuration is stored.
    CONFIG_ROOT = '/appscale/config'
  
    def __init__(self, hosts):
      """ Creates new DeploymentConfig object.
  
      Args:
        hosts: A list of ZooKeeper hosts.
      """
      self.logger = logging.getLogger(self.__class__.__name__)
      self.update_lock = Lock()
      self.state = ConfigStates.LOADING
      self.config = {}
      self.conn = KazooClient(hosts=hosts, read_only=True)
      self.conn.add_listener(self._conn_listener)
      self.conn.start()
      self.conn.ensure_path(self.CONFIG_ROOT)
      self.conn.ChildrenWatch(self.CONFIG_ROOT, func=self._update_config)
  
    def _conn_listener(self, state):
      """ Handles changes in ZooKeeper connection state.
  
      Args:
        state: A string indicating the new state.
      """
      if state == KazooState.LOST:
        self.logger.warning('ZK connection lost')
      if state == KazooState.SUSPENDED:
        self.logger.warning('ZK connection suspended')
      else:
        self.logger.info('ZK connection established')
  
    def _load_child(self, child):
      """ Fetches the data for a configuration node.
  
      Args:
        child: A string containing the ZooKeeper node to fetch.
      Returns:
        A dictionary containing configuration data.
      Raises:
        InaccessibleConfig if ZooKeeper is not accessible.
      """
      node = '/'.join([self.CONFIG_ROOT, child])
      try:
        data, _ = self.conn.retry(self.conn.get, node)
      except (KazooException, ZookeeperError):
        raise ConfigInaccessible('ZooKeeper connection not available')
      except NoNodeError:
        return {}
  
      try:
        return json.loads(data)
      except ValueError:
        self.logger.warning('Invalid deployment config: {}'.format(child))
        return {}
  
    def _update_config(self, children):
      """ Updates configuration when it changes.
  
      Args:
        children: A list of ZooKeeper nodes.
      """
      with self.update_lock:
        self.state = ConfigStates.LOADING
  
        # Ensure old sections are removed.
        self.config = {}
  
        for child in children:
          while True:
            try:
              self.config[child] = self._load_child(child)
              break
            except ConfigInaccessible as load_error:
              self.logger.warning(str(load_error))
              time.sleep(SMALL_WAIT)
  
        self.logger.info('Deployment configuration updated')
        self.state = ConfigStates.LOADED
  
    def get_config(self, section):
      """ Fetches the configuration for a given section.
  
      Args:
        section: A string specifying the section to fetch.
      Returns:
        A dictionary containing configuration data.
      Raises:
        InaccessibleConfig if ZooKeeper is inaccessible.
      """
      # If the connection is established, it should finish loading very soon.
      while (self.state == ConfigStates.LOADING and
             self.conn.state not in (KazooState.LOST, KazooState.SUSPENDED)):
        time.sleep(TINY_WAIT)
  
      if self.state != ConfigStates.LOADED:
        raise ConfigInaccessible('ZooKeeper connection not available')
  
      with self.update_lock:
        if section not in self.config:
          return {}
        return self.config[section]
  
    def close(self):
      """ Close the ZooKeeper connection. """
      self.conn.stop()
  
files: common/appscale/common/deployment_config.py
files: #!/usr/bin/env python2
  """ This script writes all the configuration files necessary to start Cassandra
  on this machine."""
  
  import argparse
  import os
  import pkgutil
  import sys
  
  from kazoo.client import KazooClient
  
  from appscale.common import appscale_info
  from appscale.common.deployment_config import DeploymentConfig
  from appscale.common.deployment_config import InvalidConfig
  
  sys.path.append(os.path.join(os.path.dirname(__file__), '../AppDB'))
  from appscale.datastore.cassandra_env.cassandra_interface import\
    CASSANDRA_INSTALL_DIR
  
  
  if __name__ == "__main__":
    parser = argparse.ArgumentParser(
      description="Creates Cassandra's Monit configuration files")
    parser.add_argument('--local-ip', required=True,
                        help='The private IP address of this machine.')
    parser.add_argument('--master-ip', required=True,
                        help='The private IP address of the database master.')
    parser.add_argument('--zk-locations', required=False,
                        help='The location of Zookeeper.')
    args = parser.parse_args()
    zk_locations = args.zk_locations if args.zk_locations else \
      appscale_info.get_zk_locations_string()
    zk_client = KazooClient(hosts=zk_locations)
    zk_client.start()
    deployment_config = DeploymentConfig(zk_client)
    cassandra_config = deployment_config.get_config('cassandra')
    if 'num_tokens' not in cassandra_config:
      raise InvalidConfig('num_tokens not specified in deployment config.')
    num_tokens = cassandra_config['num_tokens']
  
    replacements = {'APPSCALE-LOCAL': args.local_ip,
                    'APPSCALE-MASTER': args.master_ip,
                    'APPSCALE-NUM-TOKENS': num_tokens}
  
    for filename in ('cassandra.yaml', 'cassandra-env.sh'):
      dest_file_path = os.path.join(CASSANDRA_INSTALL_DIR, 'cassandra', 'conf',
                                    filename)
      contents = pkgutil.get_data('appscale.datastore.cassandra_env',
                                  'templates/{}'.format(filename))
      for key, replacement in replacements.items():
        if replacement is None:
          replacement = ''
        contents = contents.replace(key, str(replacement))
      with open(dest_file_path, 'w') as dest_file:
        dest_file.write(contents)
  
files: scripts/setup_cassandra_config_files.py
files: #!/usr/bin/env python2
  """ This script writes all the configuration files necessary to start Cassandra
  on this machine."""
  
  import argparse
  import os
  import pkgutil
  import sys
  
  from appscale.common import appscale_info
  from appscale.common.deployment_config import DeploymentConfig
  from appscale.common.deployment_config import InvalidConfig
  
  sys.path.append(os.path.join(os.path.dirname(__file__), '../AppDB'))
  from appscale.datastore.cassandra_env.cassandra_interface import\
    CASSANDRA_INSTALL_DIR
  
  
  if __name__ == "__main__":
    parser = argparse.ArgumentParser(
      description="Creates Cassandra's Monit configuration files")
    parser.add_argument('--local-ip', required=True,
                        help='The private IP address of this machine.')
    parser.add_argument('--master-ip', required=True,
                        help='The private IP address of the database master.')
    parser.add_argument('--zk-locations', required=False,
                        help='The location of Zookeeper.')
    args = parser.parse_args()
    zk_locations = args.zk_locations if args.zk_locations else \
      appscale_info.get_zk_locations_string()
    deployment_config = DeploymentConfig(zk_locations)
    cassandra_config = deployment_config.get_config('cassandra')
    if 'num_tokens' not in cassandra_config:
      raise InvalidConfig('num_tokens not specified in deployment config.')
    num_tokens = cassandra_config['num_tokens']
  
    replacements = {'APPSCALE-LOCAL': args.local_ip,
                    'APPSCALE-MASTER': args.master_ip,
                    'APPSCALE-NUM-TOKENS': num_tokens}
  
    for filename in ('cassandra.yaml', 'cassandra-env.sh'):
      dest_file_path = os.path.join(CASSANDRA_INSTALL_DIR, 'cassandra', 'conf',
                                    filename)
      contents = pkgutil.get_data('appscale.datastore.cassandra_env',
                                  'templates/{}'.format(filename))
      for key, replacement in replacements.items():
        if replacement is None:
          replacement = ''
        contents = contents.replace(key, str(replacement))
      with open(dest_file_path, 'w') as dest_file:
        dest_file.write(contents)
  
files: scripts/setup_cassandra_config_files.py