files: #!/usr/bin/env python
  #
  # Copyright 2007 Google Inc.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  #
  
  
  
  
  """An apiproxy stub that calls a remote handler via HTTP.
  
  This allows easy remote access to the App Engine datastore, and potentially any
  of the other App Engine APIs, using the same interface you use when accessing
  the service locally.
  
  An example Python script:
  ---
  from google.appengine.ext import db
  from google.appengine.ext.remote_api import remote_api_stub
  from myapp import models
  import getpass
  
  def auth_func():
    return (raw_input('Username:'), getpass.getpass('Password:'))
  
  remote_api_stub.ConfigureRemoteApi(None, '/_ah/remote_api', auth_func,
                                     'my-app.appspot.com')
  
  # Now you can access the remote datastore just as if your code was running on
  # App Engine!
  
  houses = models.House.all().fetch(100)
  for a_house in q:
    a_house.doors += 1
  db.put(houses)
  ---
  
  A few caveats:
  - Where possible, avoid iterating over queries. Fetching as many results as you
    will need is faster and more efficient. If you don't know how many results
    you need, or you need 'all of them', iterating is fine.
  - Likewise, it's a good idea to put entities in batches. Instead of calling put
    for each individual entity, accumulate them and put them in batches using
    db.put(), if you can.
  - Requests and responses are still limited to 1MB each, so if you have large
    entities or try and fetch or put many of them at once, your requests may fail.
  """
  
  
  
  
  
  
  
  
  
  import google
  import imp
  import os
  import pickle
  import random
  import sys
  import thread
  import threading
  import yaml
  import hashlib
  
  
  if os.environ.get('APPENGINE_RUNTIME') == 'python27':
    from google.appengine.api import apiproxy_rpc
    from google.appengine.api import apiproxy_stub_map
    from google.appengine.datastore import datastore_pb
    from google.appengine.ext.remote_api import remote_api_pb
    from google.appengine.ext.remote_api import remote_api_services
    from google.appengine.runtime import apiproxy_errors
  else:
    from google.appengine.api import apiproxy_rpc
    from google.appengine.api import apiproxy_stub_map
    from google.appengine.datastore import datastore_pb
    from google.appengine.ext.remote_api import remote_api_pb
    from google.appengine.ext.remote_api import remote_api_services
    from google.appengine.runtime import apiproxy_errors
  
  from google.appengine.tools import appengine_rpc
  
  
  _REQUEST_ID_HEADER = 'HTTP_X_APPENGINE_REQUEST_ID'
  
  
  class Error(Exception):
    """Base class for exceptions in this module."""
  
  
  class ConfigurationError(Error):
    """Exception for configuration errors."""
  
  
  class UnknownJavaServerError(Error):
    """Exception for exceptions returned from a Java remote_api handler."""
  
  
  def GetUserAgent():
    """Determines the value of the 'User-agent' header to use for HTTP requests.
  
    Returns:
      String containing the 'user-agent' header value, which includes the SDK
      version, the platform information, and the version of Python;
      e.g., "remote_api/1.0.1 Darwin/9.2.0 Python/2.5.2".
    """
    product_tokens = []
  
    product_tokens.append("Google-remote_api/1.0")
  
  
    product_tokens.append(appengine_rpc.GetPlatformToken())
  
  
    python_version = ".".join(str(i) for i in sys.version_info)
    product_tokens.append("Python/%s" % python_version)
  
    return " ".join(product_tokens)
  
  
  def GetSourceName():
    return "Google-remote_api-1.0"
  
  
  def HashEntity(entity):
    """Return a very-likely-unique hash of an entity."""
    return hashlib.sha1(entity.Encode()).digest()
  
  
  class TransactionData(object):
    """Encapsulates data about an individual transaction."""
  
    def __init__(self, thread_id, is_xg):
  
  
      self.thread_id = thread_id
  
  
  
      self.preconditions = {}
  
  
  
  
      self.entities = {}
  
      self.is_xg = is_xg
  
  
  class RemoteStub(object):
    """A stub for calling services on a remote server over HTTP.
  
    You can use this to stub out any service that the remote server supports.
    """
  
  
    _local = threading.local()
  
    def __init__(self, server, path, _test_stub_map=None):
      """Constructs a new RemoteStub that communicates with the specified server.
  
      Args:
        server: An instance of a subclass of
          google.appengine.tools.appengine_rpc.AbstractRpcServer.
        path: The path to the handler this stub should send requests to.
      """
  
  
      self._server = server
      self._path = path
      self._test_stub_map = _test_stub_map
  
    def _PreHookHandler(self, service, call, request, response):
      pass
  
    def _PostHookHandler(self, service, call, request, response):
      pass
  
    def MakeSyncCall(self, service, call, request, response):
      self._PreHookHandler(service, call, request, response)
      try:
        test_stub = self._test_stub_map and self._test_stub_map.GetStub(service)
        if test_stub:
  
          test_stub.MakeSyncCall(service, call, request, response)
        else:
          self._MakeRealSyncCall(service, call, request, response)
      finally:
        self._PostHookHandler(service, call, request, response)
  
    @classmethod
    def _GetRequestId(cls):
      """Returns the id of the request associated with the current thread."""
      try:
        return cls._local.request_id
      except AttributeError:
        return None
  
    @classmethod
    def _SetRequestId(cls, request_id):
      """Set the id of the request associated with the current thread."""
      cls._local.request_id = request_id
  
    def _MakeRealSyncCall(self, service, call, request, response):
      request_pb = remote_api_pb.Request()
      request_pb.set_service_name(service)
      request_pb.set_method(call)
      request_pb.set_request(request.Encode())
      if hasattr(self._local, 'request_id'):
  
  
        request_pb.set_request_id(self._local.request_id)
  
      response_pb = remote_api_pb.Response()
      encoded_request = request_pb.Encode()
      encoded_response = self._server.Send(self._path, encoded_request)
      response_pb.ParseFromString(encoded_response)
  
      if response_pb.has_application_error():
        error_pb = response_pb.application_error()
        raise apiproxy_errors.ApplicationError(error_pb.code(),
                                               error_pb.detail())
      elif response_pb.has_exception():
        raise pickle.loads(response_pb.exception())
      elif response_pb.has_java_exception():
        raise UnknownJavaServerError("An unknown error has occured in the "
                                     "Java remote_api handler for this call.")
      else:
        response.ParseFromString(response_pb.response())
  
    def CreateRPC(self):
      return apiproxy_rpc.RPC(stub=self)
  
  
  class RuntimeRemoteStub(RemoteStub):
    """ A RemoteStub that uses a separate thread for RPCs. """
  
    def CreateRPC(self):
      """ Create an RPC that can be used asynchronously. """
      # If the runtime is importing a module, fall back to the non-threaded RPC.
      # This prevents a deadlock in cases when the RealRPC thread tries to
      # acquire the import lock.
      if imp.lock_held():
        return apiproxy_rpc.RPC(stub=self)
      else:
        return apiproxy_rpc.RealRPC(stub=self)
  
  
  class RemoteDatastoreStub(RemoteStub):
    """A specialised stub for accessing the App Engine datastore remotely.
  
    A specialised stub is required because there are some datastore operations
    that preserve state between calls. This stub makes queries possible.
    Transactions on the remote datastore are unfortunately still impossible.
    """
  
    def __init__(self, server, path, default_result_count=20,
                 _test_stub_map=None):
      """Constructor.
  
      Args:
        server: The server name to connect to.
        path: The URI path on the server.
        default_result_count: The number of items to fetch, by default, in a
          datastore Query or Next operation. This affects the batch size of
          query iterators.
      """
      super(RemoteDatastoreStub, self).__init__(server, path, _test_stub_map)
      self.default_result_count = default_result_count
      self.__queries = {}
      self.__transactions = {}
  
  
  
  
      self.__next_local_cursor = 1
      self.__local_cursor_lock = threading.Lock()
      self.__next_local_tx = 1
      self.__local_tx_lock = threading.Lock()
  
    def MakeSyncCall(self, service, call, request, response):
      assert service == 'datastore_v3'
  
      explanation = []
      assert request.IsInitialized(explanation), explanation
  
      handler = getattr(self, '_Dynamic_' + call, None)
      if handler:
        handler(request, response)
      else:
        super(RemoteDatastoreStub, self).MakeSyncCall(service, call, request,
                                                      response)
  
      assert response.IsInitialized(explanation), explanation
  
    def _Dynamic_RunQuery(self, query, query_result, cursor_id = None):
      # AppScale: Removed previous check because we remotely handle transactions.
      super(RemoteDatastoreStub, self).MakeSyncCall(
            'datastore_v3', 'RunQuery', query, query_result)
  
      if cursor_id is None:
        self.__local_cursor_lock.acquire()
        try:
          cursor_id = self.__next_local_cursor
          self.__next_local_cursor += 1
        finally:
          self.__local_cursor_lock.release()
  
      if query_result.more_results():
        query.set_offset(query.offset() + query_result.result_size())
        if query.has_limit():
          query.set_limit(query.limit() - query_result.result_size())
        self.__queries[cursor_id] = query
      else:
        self.__queries[cursor_id] = None
  
  
      query_result.mutable_cursor().set_cursor(cursor_id)
  
    def _Dynamic_Next(self, next_request, query_result):
      assert next_request.offset() == 0
      cursor_id = next_request.cursor().cursor()
      if cursor_id not in self.__queries:
        raise apiproxy_errors.ApplicationError(datastore_pb.Error.BAD_REQUEST,
                                               'Cursor %d not found' % cursor_id)
      query = self.__queries[cursor_id]
  
      if query is None:
  
        query_result.set_more_results(False)
        return
      else:
        if next_request.has_count():
          query.set_count(next_request.count())
        else:
          query.clear_count()
  
      self._Dynamic_RunQuery(query, query_result, cursor_id)
  
  
  
  
      query_result.set_skipped_results(0)
  
    def _Dynamic_Get(self, get_request, get_response):
      txid = None
      if get_request.has_transaction():
  
        txid = get_request.transaction().handle()
        txdata = self.__transactions[txid]
        assert (txdata.thread_id ==
            thread.get_ident()), "Transactions are single-threaded."
  
  
        keys = [(k, k.Encode()) for k in get_request.key_list()]
  
  
        new_request = datastore_pb.GetRequest()
        for key, enckey in keys:
          if enckey not in txdata.entities:
            new_request.add_key().CopyFrom(key)
      else:
        new_request = get_request
  
      if new_request.key_size() > 0:
        super(RemoteDatastoreStub, self).MakeSyncCall(
            'datastore_v3', 'Get', new_request, get_response)
  
      if txid is not None:
  
        newkeys = new_request.key_list()
        entities = get_response.entity_list()
        for key, entity in zip(newkeys, entities):
          entity_hash = None
          if entity.has_entity():
            entity_hash = HashEntity(entity.entity())
          txdata.preconditions[key.Encode()] = (key, entity_hash)
  
  
  
  
  
        new_response = datastore_pb.GetResponse()
        it = iter(get_response.entity_list())
        for key, enckey in keys:
          if enckey in txdata.entities:
            cached_entity = txdata.entities[enckey][1]
            if cached_entity:
              new_response.add_entity().mutable_entity().CopyFrom(cached_entity)
            else:
              new_response.add_entity()
          else:
            new_entity = it.next()
            if new_entity.has_entity():
              assert new_entity.entity().key() == key
              new_response.add_entity().CopyFrom(new_entity)
            else:
              new_response.add_entity()
        get_response.CopyFrom(new_response)
  
    def _Dynamic_Put(self, put_request, put_response):
      if put_request.has_transaction():
        entities = put_request.entity_list()
  
  
        requires_id = lambda x: x.id() == 0 and not x.has_name()
        new_ents = [e for e in entities
                    if requires_id(e.key().path().element_list()[-1])]
        id_request = datastore_pb.PutRequest()
  
        txid = put_request.transaction().handle()
        txdata = self.__transactions[txid]
        assert (txdata.thread_id ==
            thread.get_ident()), "Transactions are single-threaded."
        if new_ents:
          for ent in new_ents:
            e = id_request.add_entity()
            e.mutable_key().CopyFrom(ent.key())
            e.mutable_entity_group()
          id_response = datastore_pb.PutResponse()
  
  
  
          if txdata.is_xg:
            rpc_name = 'GetIDsXG'
          else:
            rpc_name = 'GetIDs'
          super(RemoteDatastoreStub, self).MakeSyncCall(
              'remote_datastore', rpc_name, id_request, id_response)
          assert id_request.entity_size() == id_response.key_size()
          for key, ent in zip(id_response.key_list(), new_ents):
            ent.mutable_key().CopyFrom(key)
            ent.mutable_entity_group().add_element().CopyFrom(
                key.path().element(0))
  
        for entity in entities:
          txdata.entities[entity.key().Encode()] = (entity.key(), entity)
          put_response.add_key().CopyFrom(entity.key())
      else:
        super(RemoteDatastoreStub, self).MakeSyncCall(
            'datastore_v3', 'Put', put_request, put_response)
  
    def _Dynamic_Delete(self, delete_request, response):
      if delete_request.has_transaction():
        txid = delete_request.transaction().handle()
        txdata = self.__transactions[txid]
        assert (txdata.thread_id ==
            thread.get_ident()), "Transactions are single-threaded."
        for key in delete_request.key_list():
          txdata.entities[key.Encode()] = (key, None)
      else:
        super(RemoteDatastoreStub, self).MakeSyncCall(
            'datastore_v3', 'Delete', delete_request, response)
  
    def _Dynamic_BeginTransaction(self, request, transaction):
      self.__local_tx_lock.acquire()
      try:
        txid = self.__next_local_tx
        self.__transactions[txid] = TransactionData(thread.get_ident(),
                                                    request.allow_multiple_eg())
        self.__next_local_tx += 1
      finally:
        self.__local_tx_lock.release()
      transaction.set_handle(txid)
      transaction.set_app(request.app())
  
    def _Dynamic_Commit(self, transaction, transaction_response):
      txid = transaction.handle()
      if txid not in self.__transactions:
        raise apiproxy_errors.ApplicationError(
            datastore_pb.Error.BAD_REQUEST,
            'Transaction %d not found.' % (txid,))
  
      txdata = self.__transactions[txid]
      assert (txdata.thread_id ==
          thread.get_ident()), "Transactions are single-threaded."
      del self.__transactions[txid]
  
      tx = remote_api_pb.TransactionRequest()
      tx.set_allow_multiple_eg(txdata.is_xg)
      for key, hash in txdata.preconditions.values():
        precond = tx.add_precondition()
        precond.mutable_key().CopyFrom(key)
        if hash:
          precond.set_hash(hash)
  
      puts = tx.mutable_puts()
      deletes = tx.mutable_deletes()
      for key, entity in txdata.entities.values():
        if entity:
          puts.add_entity().CopyFrom(entity)
        else:
          deletes.add_key().CopyFrom(key)
  
  
      super(RemoteDatastoreStub, self).MakeSyncCall(
          'remote_datastore', 'Transaction',
          tx, datastore_pb.PutResponse())
  
    def _Dynamic_Rollback(self, transaction, transaction_response):
      txid = transaction.handle()
      self.__local_tx_lock.acquire()
      try:
        if txid not in self.__transactions:
          raise apiproxy_errors.ApplicationError(
              datastore_pb.Error.BAD_REQUEST,
              'Transaction %d not found.' % (txid,))
  
        txdata = self.__transactions[txid]
        assert (txdata.thread_id ==
            thread.get_ident()), "Transactions are single-threaded."
        del self.__transactions[txid]
      finally:
        self.__local_tx_lock.release()
  
    def _Dynamic_CreateIndex(self, index, id_response):
      raise apiproxy_errors.CapabilityDisabledError(
          'The remote datastore does not support index manipulation.')
  
    def _Dynamic_UpdateIndex(self, index, void):
      raise apiproxy_errors.CapabilityDisabledError(
          'The remote datastore does not support index manipulation.')
  
    def _Dynamic_DeleteIndex(self, index, void):
      raise apiproxy_errors.CapabilityDisabledError(
          'The remote datastore does not support index manipulation.')
  
  
  class RuntimeDatastoreStub(RemoteDatastoreStub):
    """ A datastore stub that uses a separate thread for RPCs. """
  
    def CreateRPC(self):
      """ Create an RPC that can be used asynchronously. """
      # If the runtime is importing a module, fall back to the non-threaded RPC.
      # This prevents a deadlock in cases when the RealRPC thread tries to
      # acquire the import lock.
      if imp.lock_held():
        return apiproxy_rpc.RPC(stub=self)
      else:
        return apiproxy_rpc.RealRPC(stub=self)
  
  
  ALL_SERVICES = set(remote_api_services.SERVICE_PB_MAP)
  
  
  def GetRemoteAppIdFromServer(server, path, remote_token=None):
    """Return the app id from a connection to an existing server.
  
    Args:
      server: An appengine_rpc.AbstractRpcServer
      path: The path to the remote_api handler for your app
        (for example, '/_ah/remote_api').
      remote_token: Token to validate that the response was to this request.
    Returns:
      App ID as reported by the remote server.
    Raises:
      ConfigurationError: The server returned an invalid response.
    """
    if not remote_token:
      random.seed()
      remote_token = str(random.random())[2:]
    remote_token = str(remote_token)
    urlargs = {'rtok': remote_token}
    response = server.Send(path, payload=None, **urlargs)
    if not response.startswith('{'):
      raise ConfigurationError(
          'Invalid response recieved from server: %s' % response)
    app_info = yaml.load(response)
    if not app_info or 'rtok' not in app_info or 'app_id' not in app_info:
      raise ConfigurationError('Error parsing app_id lookup response')
    if str(app_info['rtok']) != remote_token:
      raise ConfigurationError('Token validation failed during app_id lookup. '
                               '(sent %s, got %s)' % (repr(remote_token),
                                                      repr(app_info['rtok'])))
    return app_info['app_id']
  
  
  def ConfigureRemoteApiFromServer(server, path, app_id, services=None,
                                   default_auth_domain=None,
                                   use_remote_datastore=True,
                                   use_async_rpc=False,
                                   external_server=None):
    """Does necessary setup to allow easy remote access to App Engine APIs.
  
    Args:
      server: An AbstractRpcServer
      path: The path to the remote_api handler for your app
        (for example, '/_ah/remote_api').
      app_id: The app_id of your app, as declared in app.yaml.
      services: A list of services to set up stubs for. If specified, only those
        services are configured; by default all supported services are configured.
      default_auth_domain: The authentication domain to use by default.
      use_remote_datastore: Whether to use RemoteDatastoreStub instead of passing
        through datastore requests. RemoteDatastoreStub batches transactional
        datastore requests since, in production, datastore requires are scoped to
        a single request.
      use_async_rpc: A boolean indicating whether or not to make RPC calls in a
        separate thread.
      external_server: An AbstractRpcServer specifying the location of an
        external API server.
  
    Raises:
      urllib2.HTTPError: if app_id is not provided and there is an error while
        retrieving it.
      ConfigurationError: if there is a error configuring the Remote API.
    """
    if services is None:
      services = set(ALL_SERVICES)
    else:
      services = set(services)
      unsupported = services.difference(ALL_SERVICES)
      if unsupported:
        raise ConfigurationError('Unsupported service(s): %s'
                                 % (', '.join(unsupported),))
  
    os.environ['APPLICATION_ID'] = app_id
    os.environ.setdefault('AUTH_DOMAIN', default_auth_domain or 'gmail.com')
    apiproxy_stub_map.apiproxy = apiproxy_stub_map.APIProxyStubMap()
    if 'datastore_v3' in services and use_remote_datastore:
      services.remove('datastore_v3')
      if use_async_rpc:
        datastore_stub = RuntimeDatastoreStub(server, path)
      else:
        datastore_stub = RemoteDatastoreStub(server, path)
      apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', datastore_stub)
  
    if use_async_rpc:
      stub_type = RuntimeRemoteStub
    else:
      stub_type = RemoteStub
  
    stub = stub_type(server, path)
    external_stub = None
    if external_server is not None:
      external_stub = stub_type(external_server, path)
  
    for service in services:
      if service == 'app_identity_service' and external_stub is not None:
        apiproxy_stub_map.apiproxy.RegisterStub(service, external_stub)
      else:
        apiproxy_stub_map.apiproxy.RegisterStub(service, stub)
  
  
  def GetRemoteAppId(servername,
                     path,
                     auth_func,
                     rpc_server_factory=appengine_rpc.HttpRpcServer,
                     rtok=None,
                     secure=False,
                     save_cookies=False):
    """Get the remote appid as reported at servername/path.
  
    This will also return an AbstractRpcServer server, which can be used with
    ConfigureRemoteApiFromServer.
  
    Args:
      servername: The hostname your app is deployed on.
      path: The path to the remote_api handler for your app
        (for example, '/_ah/remote_api').
      auth_func: A function that takes no arguments and returns a
        (username, password) tuple. This will be called if your application
        requires authentication to access the remote_api handler (it should!)
        and you do not already have a valid auth cookie.
        <app_id>.appspot.com.
      rpc_server_factory: A factory to construct the rpc server for the datastore.
      rtok: The validation token to sent with app_id lookups. If None, a random
        token is used.
      secure: Use SSL when communicating with the server.
      save_cookies: Forwarded to rpc_server_factory function.
  
    Returns:
      (app_id, server): The application ID and an AbstractRpcServer.
    """
    server = rpc_server_factory(servername, auth_func, GetUserAgent(),
                                GetSourceName(), save_cookies=save_cookies,
                                debug_data=False, secure=secure)
    app_id = GetRemoteAppIdFromServer(server, path, rtok)
    return app_id, server
  
  
  def ConfigureRemoteApi(app_id,
                         path,
                         auth_func,
                         servername=None,
                         rpc_server_factory=appengine_rpc.HttpRpcServer,
                         rtok=None,
                         secure=False,
                         services=None,
                         default_auth_domain=None,
                         save_cookies=False,
                         use_remote_datastore=True,
                         use_async_rpc=False,
                         external_api_server=None):
    """Does necessary setup to allow easy remote access to App Engine APIs.
  
    Either servername must be provided or app_id must not be None.  If app_id
    is None and a servername is provided, this function will send a request
    to the server to retrieve the app_id.
  
    Note that if the app_id is specified, the internal appid must be used;
    this may include a partition and a domain. It is often easier to let
    remote_api_stub retreive the app_id automatically.
  
  
    Args:
      app_id: The app_id of your app, as declared in app.yaml, or None.
      path: The path to the remote_api handler for your app
        (for example, '/_ah/remote_api').
      auth_func: A function that takes no arguments and returns a
        (username, password) tuple. This will be called if your application
        requires authentication to access the remote_api handler (it should!)
        and you do not already have a valid auth cookie.
      servername: The hostname your app is deployed on. Defaults to
        <app_id>.appspot.com.
      rpc_server_factory: A factory to construct the rpc server for the datastore.
      rtok: The validation token to sent with app_id lookups. If None, a random
        token is used.
      secure: Use SSL when communicating with the server.
      services: A list of services to set up stubs for. If specified, only those
        services are configured; by default all supported services are configured.
      default_auth_domain: The authentication domain to use by default.
      save_cookies: Forwarded to rpc_server_factory function.
      use_remote_datastore: Whether to use RemoteDatastoreStub instead of passing
        through datastore requests. RemoteDatastoreStub batches transactional
        datastore requests since, in production, datastore requires are scoped to
        a single request.
      use_async_rpc: A boolean indicating whether or not to make RPC calls in a
        separate thread.
      external_api_server: A string specifying the location of an external API
        server.
  
    Returns:
      server, the server created by rpc_server_factory, which may be useful for
        calling the application directly.
  
    Raises:
      urllib2.HTTPError: if app_id is not provided and there is an error while
        retrieving it.
      ConfigurationError: if there is a error configuring the DatstoreFileStub.
    """
    if not servername and not app_id:
      raise ConfigurationError('app_id or servername required')
    if not servername:
      servername = '%s.appspot.com' % (app_id,)
    server = rpc_server_factory(servername, auth_func, GetUserAgent(),
                                GetSourceName(), save_cookies=save_cookies,
                                debug_data=False, secure=secure)
  
    if external_api_server is None:
      external_server = server
    else:
      external_server = rpc_server_factory(
        external_api_server, auth_func, GetUserAgent(), GetSourceName(),
        save_cookies=save_cookies, debug_data=False, secure=secure)
  
    if not app_id:
      app_id = GetRemoteAppIdFromServer(server, path, rtok)
  
    ConfigureRemoteApiFromServer(server, path, app_id, services,
                                 default_auth_domain, use_remote_datastore,
                                 use_async_rpc, external_server)
    return server
  
  
  def MaybeInvokeAuthentication():
    """Sends an empty request through to the configured end-point.
  
    If authentication is necessary, this will cause the rpc_server to invoke
    interactive authentication.
    """
    datastore_stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3')
    if isinstance(datastore_stub, RemoteStub):
      datastore_stub._server.Send(datastore_stub._path, payload=None)
    else:
      raise ConfigurationError('remote_api is not configured.')
  
  
  
  ConfigureRemoteDatastore = ConfigureRemoteApi
  
files: AppServer/google/appengine/ext/remote_api/remote_api_stub.py
files: #!/usr/bin/env python
  #
  # Copyright 2007 Google Inc.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  #
  
  
  
  
  """An apiproxy stub that calls a remote handler via HTTP.
  
  This allows easy remote access to the App Engine datastore, and potentially any
  of the other App Engine APIs, using the same interface you use when accessing
  the service locally.
  
  An example Python script:
  ---
  from google.appengine.ext import db
  from google.appengine.ext.remote_api import remote_api_stub
  from myapp import models
  import getpass
  
  def auth_func():
    return (raw_input('Username:'), getpass.getpass('Password:'))
  
  remote_api_stub.ConfigureRemoteApi(None, '/_ah/remote_api', auth_func,
                                     'my-app.appspot.com')
  
  # Now you can access the remote datastore just as if your code was running on
  # App Engine!
  
  houses = models.House.all().fetch(100)
  for a_house in q:
    a_house.doors += 1
  db.put(houses)
  ---
  
  A few caveats:
  - Where possible, avoid iterating over queries. Fetching as many results as you
    will need is faster and more efficient. If you don't know how many results
    you need, or you need 'all of them', iterating is fine.
  - Likewise, it's a good idea to put entities in batches. Instead of calling put
    for each individual entity, accumulate them and put them in batches using
    db.put(), if you can.
  - Requests and responses are still limited to 1MB each, so if you have large
    entities or try and fetch or put many of them at once, your requests may fail.
  """
  
  
  
  
  
  
  
  
  
  import google
  import imp
  import os
  import pickle
  import random
  import sys
  import thread
  import threading
  import yaml
  import hashlib
  
  
  if os.environ.get('APPENGINE_RUNTIME') == 'python27':
    from google.appengine.api import apiproxy_rpc
    from google.appengine.api import apiproxy_stub_map
    from google.appengine.datastore import datastore_pb
    from google.appengine.ext.remote_api import remote_api_pb
    from google.appengine.ext.remote_api import remote_api_services
    from google.appengine.runtime import apiproxy_errors
  else:
    from google.appengine.api import apiproxy_rpc
    from google.appengine.api import apiproxy_stub_map
    from google.appengine.datastore import datastore_pb
    from google.appengine.ext.remote_api import remote_api_pb
    from google.appengine.ext.remote_api import remote_api_services
    from google.appengine.runtime import apiproxy_errors
  
  from google.appengine.tools import appengine_rpc
  
  
  _REQUEST_ID_HEADER = 'HTTP_X_APPENGINE_REQUEST_ID'
  
  
  class Error(Exception):
    """Base class for exceptions in this module."""
  
  
  class ConfigurationError(Error):
    """Exception for configuration errors."""
  
  
  class UnknownJavaServerError(Error):
    """Exception for exceptions returned from a Java remote_api handler."""
  
  
  def GetUserAgent():
    """Determines the value of the 'User-agent' header to use for HTTP requests.
  
    Returns:
      String containing the 'user-agent' header value, which includes the SDK
      version, the platform information, and the version of Python;
      e.g., "remote_api/1.0.1 Darwin/9.2.0 Python/2.5.2".
    """
    product_tokens = []
  
    product_tokens.append("Google-remote_api/1.0")
  
  
    product_tokens.append(appengine_rpc.GetPlatformToken())
  
  
    python_version = ".".join(str(i) for i in sys.version_info)
    product_tokens.append("Python/%s" % python_version)
  
    return " ".join(product_tokens)
  
  
  def GetSourceName():
    return "Google-remote_api-1.0"
  
  
  def HashEntity(entity):
    """Return a very-likely-unique hash of an entity."""
    return hashlib.sha1(entity.Encode()).digest()
  
  
  class TransactionData(object):
    """Encapsulates data about an individual transaction."""
  
    def __init__(self, thread_id, is_xg):
  
  
      self.thread_id = thread_id
  
  
  
      self.preconditions = {}
  
  
  
  
      self.entities = {}
  
      self.is_xg = is_xg
  
  
  class RemoteStub(object):
    """A stub for calling services on a remote server over HTTP.
  
    You can use this to stub out any service that the remote server supports.
    """
  
  
    _local = threading.local()
  
    def __init__(self, server, path, _test_stub_map=None):
      """Constructs a new RemoteStub that communicates with the specified server.
  
      Args:
        server: An instance of a subclass of
          google.appengine.tools.appengine_rpc.AbstractRpcServer.
        path: The path to the handler this stub should send requests to.
      """
  
  
      self._server = server
      self._path = path
      self._test_stub_map = _test_stub_map
  
    def _PreHookHandler(self, service, call, request, response):
      pass
  
    def _PostHookHandler(self, service, call, request, response):
      pass
  
    def MakeSyncCall(self, service, call, request, response):
      self._PreHookHandler(service, call, request, response)
      try:
        test_stub = self._test_stub_map and self._test_stub_map.GetStub(service)
        if test_stub:
  
          test_stub.MakeSyncCall(service, call, request, response)
        else:
          self._MakeRealSyncCall(service, call, request, response)
      finally:
        self._PostHookHandler(service, call, request, response)
  
    @classmethod
    def _GetRequestId(cls):
      """Returns the id of the request associated with the current thread."""
      try:
        return cls._local.request_id
      except AttributeError:
        return None
  
    @classmethod
    def _SetRequestId(cls, request_id):
      """Set the id of the request associated with the current thread."""
      cls._local.request_id = request_id
  
    def _MakeRealSyncCall(self, service, call, request, response):
      request_pb = remote_api_pb.Request()
      request_pb.set_service_name(service)
      request_pb.set_method(call)
      request_pb.set_request(request.Encode())
      if hasattr(self._local, 'request_id'):
  
  
        request_pb.set_request_id(self._local.request_id)
  
      response_pb = remote_api_pb.Response()
      encoded_request = request_pb.Encode()
      encoded_response = self._server.Send(self._path, encoded_request)
      response_pb.ParseFromString(encoded_response)
  
      if response_pb.has_application_error():
        error_pb = response_pb.application_error()
        raise apiproxy_errors.ApplicationError(error_pb.code(),
                                               error_pb.detail())
      elif response_pb.has_exception():
        raise pickle.loads(response_pb.exception())
      elif response_pb.has_java_exception():
        raise UnknownJavaServerError("An unknown error has occured in the "
                                     "Java remote_api handler for this call.")
      else:
        response.ParseFromString(response_pb.response())
  
    def CreateRPC(self):
      return apiproxy_rpc.RPC(stub=self)
  
  
  class RuntimeRemoteStub(RemoteStub):
    """ A RemoteStub that uses a separate thread for RPCs. """
  
    def CreateRPC(self):
      """ Create an RPC that can be used asynchronously. """
      # If the runtime is importing a module, fall back to the non-threaded RPC.
      # This prevents a deadlock in cases when the RealRPC thread tries to
      # acquire the import lock.
      if imp.lock_held():
        return apiproxy_rpc.RPC(stub=self)
      else:
        return apiproxy_rpc.RealRPC(stub=self)
  
  
  class RemoteDatastoreStub(RemoteStub):
    """A specialised stub for accessing the App Engine datastore remotely.
  
    A specialised stub is required because there are some datastore operations
    that preserve state between calls. This stub makes queries possible.
    Transactions on the remote datastore are unfortunately still impossible.
    """
  
    def __init__(self, server, path, default_result_count=20,
                 _test_stub_map=None):
      """Constructor.
  
      Args:
        server: The server name to connect to.
        path: The URI path on the server.
        default_result_count: The number of items to fetch, by default, in a
          datastore Query or Next operation. This affects the batch size of
          query iterators.
      """
      super(RemoteDatastoreStub, self).__init__(server, path, _test_stub_map)
      self.default_result_count = default_result_count
      self.__queries = {}
      self.__transactions = {}
  
  
  
  
      self.__next_local_cursor = 1
      self.__local_cursor_lock = threading.Lock()
      self.__next_local_tx = 1
      self.__local_tx_lock = threading.Lock()
  
    def MakeSyncCall(self, service, call, request, response):
      assert service == 'datastore_v3'
  
      explanation = []
      assert request.IsInitialized(explanation), explanation
  
      handler = getattr(self, '_Dynamic_' + call, None)
      if handler:
        handler(request, response)
      else:
        super(RemoteDatastoreStub, self).MakeSyncCall(service, call, request,
                                                      response)
  
      assert response.IsInitialized(explanation), explanation
  
    def _Dynamic_RunQuery(self, query, query_result, cursor_id = None):
      # AppScale: Removed previous check because we remotely handle transactions.
      super(RemoteDatastoreStub, self).MakeSyncCall(
            'datastore_v3', 'RunQuery', query, query_result)
  
      if cursor_id is None:
        self.__local_cursor_lock.acquire()
        try:
          cursor_id = self.__next_local_cursor
          self.__next_local_cursor += 1
        finally:
          self.__local_cursor_lock.release()
  
      if query_result.more_results():
        query.set_offset(query.offset() + query_result.result_size())
        if query.has_limit():
          query.set_limit(query.limit() - query_result.result_size())
        self.__queries[cursor_id] = query
      else:
        self.__queries[cursor_id] = None
  
  
      query_result.mutable_cursor().set_cursor(cursor_id)
  
    def _Dynamic_Next(self, next_request, query_result):
      assert next_request.offset() == 0
      cursor_id = next_request.cursor().cursor()
      if cursor_id not in self.__queries:
        raise apiproxy_errors.ApplicationError(datastore_pb.Error.BAD_REQUEST,
                                               'Cursor %d not found' % cursor_id)
      query = self.__queries[cursor_id]
  
      if query is None:
  
        query_result.set_more_results(False)
        return
      else:
        if next_request.has_count():
          query.set_count(next_request.count())
        else:
          query.clear_count()
  
      self._Dynamic_RunQuery(query, query_result, cursor_id)
  
  
  
  
      query_result.set_skipped_results(0)
  
    def _Dynamic_Get(self, get_request, get_response):
      txid = None
      if get_request.has_transaction():
  
        txid = get_request.transaction().handle()
        txdata = self.__transactions[txid]
        assert (txdata.thread_id ==
            thread.get_ident()), "Transactions are single-threaded."
  
  
        keys = [(k, k.Encode()) for k in get_request.key_list()]
  
  
        new_request = datastore_pb.GetRequest()
        for key, enckey in keys:
          if enckey not in txdata.entities:
            new_request.add_key().CopyFrom(key)
      else:
        new_request = get_request
  
      if new_request.key_size() > 0:
        super(RemoteDatastoreStub, self).MakeSyncCall(
            'datastore_v3', 'Get', new_request, get_response)
  
      if txid is not None:
  
        newkeys = new_request.key_list()
        entities = get_response.entity_list()
        for key, entity in zip(newkeys, entities):
          entity_hash = None
          if entity.has_entity():
            entity_hash = HashEntity(entity.entity())
          txdata.preconditions[key.Encode()] = (key, entity_hash)
  
  
  
  
  
        new_response = datastore_pb.GetResponse()
        it = iter(get_response.entity_list())
        for key, enckey in keys:
          if enckey in txdata.entities:
            cached_entity = txdata.entities[enckey][1]
            if cached_entity:
              new_response.add_entity().mutable_entity().CopyFrom(cached_entity)
            else:
              new_response.add_entity()
          else:
            new_entity = it.next()
            if new_entity.has_entity():
              assert new_entity.entity().key() == key
              new_response.add_entity().CopyFrom(new_entity)
            else:
              new_response.add_entity()
        get_response.CopyFrom(new_response)
  
    def _Dynamic_Put(self, put_request, put_response):
      if put_request.has_transaction():
        entities = put_request.entity_list()
  
  
        requires_id = lambda x: x.id() == 0 and not x.has_name()
        new_ents = [e for e in entities
                    if requires_id(e.key().path().element_list()[-1])]
        id_request = datastore_pb.PutRequest()
  
        txid = put_request.transaction().handle()
        txdata = self.__transactions[txid]
        assert (txdata.thread_id ==
            thread.get_ident()), "Transactions are single-threaded."
        if new_ents:
          for ent in new_ents:
            e = id_request.add_entity()
            e.mutable_key().CopyFrom(ent.key())
            e.mutable_entity_group()
          id_response = datastore_pb.PutResponse()
  
  
  
          if txdata.is_xg:
            rpc_name = 'GetIDsXG'
          else:
            rpc_name = 'GetIDs'
          super(RemoteDatastoreStub, self).MakeSyncCall(
              'remote_datastore', rpc_name, id_request, id_response)
          assert id_request.entity_size() == id_response.key_size()
          for key, ent in zip(id_response.key_list(), new_ents):
            ent.mutable_key().CopyFrom(key)
            ent.mutable_entity_group().add_element().CopyFrom(
                key.path().element(0))
  
        for entity in entities:
          txdata.entities[entity.key().Encode()] = (entity.key(), entity)
          put_response.add_key().CopyFrom(entity.key())
      else:
        super(RemoteDatastoreStub, self).MakeSyncCall(
            'datastore_v3', 'Put', put_request, put_response)
  
    def _Dynamic_Delete(self, delete_request, response):
      if delete_request.has_transaction():
        txid = delete_request.transaction().handle()
        txdata = self.__transactions[txid]
        assert (txdata.thread_id ==
            thread.get_ident()), "Transactions are single-threaded."
        for key in delete_request.key_list():
          txdata.entities[key.Encode()] = (key, None)
      else:
        super(RemoteDatastoreStub, self).MakeSyncCall(
            'datastore_v3', 'Delete', delete_request, response)
  
    def _Dynamic_BeginTransaction(self, request, transaction):
      self.__local_tx_lock.acquire()
      try:
        txid = self.__next_local_tx
        self.__transactions[txid] = TransactionData(thread.get_ident(),
                                                    request.allow_multiple_eg())
        self.__next_local_tx += 1
      finally:
        self.__local_tx_lock.release()
      transaction.set_handle(txid)
      transaction.set_app(request.app())
  
    def _Dynamic_Commit(self, transaction, transaction_response):
      txid = transaction.handle()
      if txid not in self.__transactions:
        raise apiproxy_errors.ApplicationError(
            datastore_pb.Error.BAD_REQUEST,
            'Transaction %d not found.' % (txid,))
  
      txdata = self.__transactions[txid]
      assert (txdata.thread_id ==
          thread.get_ident()), "Transactions are single-threaded."
      del self.__transactions[txid]
  
      tx = remote_api_pb.TransactionRequest()
      tx.set_allow_multiple_eg(txdata.is_xg)
      for key, hash in txdata.preconditions.values():
        precond = tx.add_precondition()
        precond.mutable_key().CopyFrom(key)
        if hash:
          precond.set_hash(hash)
  
      puts = tx.mutable_puts()
      deletes = tx.mutable_deletes()
      for key, entity in txdata.entities.values():
        if entity:
          puts.add_entity().CopyFrom(entity)
        else:
          deletes.add_key().CopyFrom(key)
  
  
      super(RemoteDatastoreStub, self).MakeSyncCall(
          'remote_datastore', 'Transaction',
          tx, datastore_pb.PutResponse())
  
    def _Dynamic_Rollback(self, transaction, transaction_response):
      txid = transaction.handle()
      self.__local_tx_lock.acquire()
      try:
        if txid not in self.__transactions:
          raise apiproxy_errors.ApplicationError(
              datastore_pb.Error.BAD_REQUEST,
              'Transaction %d not found.' % (txid,))
  
        txdata = self.__transactions[txid]
        assert (txdata.thread_id ==
            thread.get_ident()), "Transactions are single-threaded."
        del self.__transactions[txid]
      finally:
        self.__local_tx_lock.release()
  
    def _Dynamic_CreateIndex(self, index, id_response):
      raise apiproxy_errors.CapabilityDisabledError(
          'The remote datastore does not support index manipulation.')
  
    def _Dynamic_UpdateIndex(self, index, void):
      raise apiproxy_errors.CapabilityDisabledError(
          'The remote datastore does not support index manipulation.')
  
    def _Dynamic_DeleteIndex(self, index, void):
      raise apiproxy_errors.CapabilityDisabledError(
          'The remote datastore does not support index manipulation.')
  
  
  class RuntimeDatastoreStub(RemoteDatastoreStub):
    """ A datastore stub that uses a separate thread for RPCs. """
  
    def CreateRPC(self):
      """ Create an RPC that can be used asynchronously. """
      # If the runtime is importing a module, fall back to the non-threaded RPC.
      # This prevents a deadlock in cases when the RealRPC thread tries to
      # acquire the import lock.
      if imp.lock_held():
        return apiproxy_rpc.RPC(stub=self)
      else:
        return apiproxy_rpc.RealRPC(stub=self)
  
  
  ALL_SERVICES = set(remote_api_services.SERVICE_PB_MAP)
  
  
  def GetRemoteAppIdFromServer(server, path, remote_token=None):
    """Return the app id from a connection to an existing server.
  
    Args:
      server: An appengine_rpc.AbstractRpcServer
      path: The path to the remote_api handler for your app
        (for example, '/_ah/remote_api').
      remote_token: Token to validate that the response was to this request.
    Returns:
      App ID as reported by the remote server.
    Raises:
      ConfigurationError: The server returned an invalid response.
    """
    if not remote_token:
      random.seed()
      remote_token = str(random.random())[2:]
    remote_token = str(remote_token)
    urlargs = {'rtok': remote_token}
    response = server.Send(path, payload=None, **urlargs)
    if not response.startswith('{'):
      raise ConfigurationError(
          'Invalid response recieved from server: %s' % response)
    app_info = yaml.load(response)
    if not app_info or 'rtok' not in app_info or 'app_id' not in app_info:
      raise ConfigurationError('Error parsing app_id lookup response')
    if str(app_info['rtok']) != remote_token:
      raise ConfigurationError('Token validation failed during app_id lookup. '
                               '(sent %s, got %s)' % (repr(remote_token),
                                                      repr(app_info['rtok'])))
    return app_info['app_id']
  
  
  def ConfigureRemoteApiFromServer(server, path, app_id, services=None,
                                   default_auth_domain=None,
                                   use_remote_datastore=True,
                                   use_async_rpc=False):
    """Does necessary setup to allow easy remote access to App Engine APIs.
  
    Args:
      server: An AbstractRpcServer
      path: The path to the remote_api handler for your app
        (for example, '/_ah/remote_api').
      app_id: The app_id of your app, as declared in app.yaml.
      services: A list of services to set up stubs for. If specified, only those
        services are configured; by default all supported services are configured.
      default_auth_domain: The authentication domain to use by default.
      use_remote_datastore: Whether to use RemoteDatastoreStub instead of passing
        through datastore requests. RemoteDatastoreStub batches transactional
        datastore requests since, in production, datastore requires are scoped to
        a single request.
      use_async_rpc: A boolean indicating whether or not to make RPC calls in a
        separate thread.
  
    Raises:
      urllib2.HTTPError: if app_id is not provided and there is an error while
        retrieving it.
      ConfigurationError: if there is a error configuring the Remote API.
    """
    if services is None:
      services = set(ALL_SERVICES)
    else:
      services = set(services)
      unsupported = services.difference(ALL_SERVICES)
      if unsupported:
        raise ConfigurationError('Unsupported service(s): %s'
                                 % (', '.join(unsupported),))
  
    os.environ['APPLICATION_ID'] = app_id
    os.environ.setdefault('AUTH_DOMAIN', default_auth_domain or 'gmail.com')
    apiproxy_stub_map.apiproxy = apiproxy_stub_map.APIProxyStubMap()
    if 'datastore_v3' in services and use_remote_datastore:
      services.remove('datastore_v3')
      if use_async_rpc:
        datastore_stub = RuntimeDatastoreStub(server, path)
      else:
        datastore_stub = RemoteDatastoreStub(server, path)
      apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', datastore_stub)
  
    if use_async_rpc:
      stub = RuntimeRemoteStub(server, path)
    else:
      stub = RemoteStub(server, path)
    for service in services:
      apiproxy_stub_map.apiproxy.RegisterStub(service, stub)
  
  
  def GetRemoteAppId(servername,
                     path,
                     auth_func,
                     rpc_server_factory=appengine_rpc.HttpRpcServer,
                     rtok=None,
                     secure=False,
                     save_cookies=False):
    """Get the remote appid as reported at servername/path.
  
    This will also return an AbstractRpcServer server, which can be used with
    ConfigureRemoteApiFromServer.
  
    Args:
      servername: The hostname your app is deployed on.
      path: The path to the remote_api handler for your app
        (for example, '/_ah/remote_api').
      auth_func: A function that takes no arguments and returns a
        (username, password) tuple. This will be called if your application
        requires authentication to access the remote_api handler (it should!)
        and you do not already have a valid auth cookie.
        <app_id>.appspot.com.
      rpc_server_factory: A factory to construct the rpc server for the datastore.
      rtok: The validation token to sent with app_id lookups. If None, a random
        token is used.
      secure: Use SSL when communicating with the server.
      save_cookies: Forwarded to rpc_server_factory function.
  
    Returns:
      (app_id, server): The application ID and an AbstractRpcServer.
    """
    server = rpc_server_factory(servername, auth_func, GetUserAgent(),
                                GetSourceName(), save_cookies=save_cookies,
                                debug_data=False, secure=secure)
    app_id = GetRemoteAppIdFromServer(server, path, rtok)
    return app_id, server
  
  
  def ConfigureRemoteApi(app_id,
                         path,
                         auth_func,
                         servername=None,
                         rpc_server_factory=appengine_rpc.HttpRpcServer,
                         rtok=None,
                         secure=False,
                         services=None,
                         default_auth_domain=None,
                         save_cookies=False,
                         use_remote_datastore=True,
                         use_async_rpc=False):
    """Does necessary setup to allow easy remote access to App Engine APIs.
  
    Either servername must be provided or app_id must not be None.  If app_id
    is None and a servername is provided, this function will send a request
    to the server to retrieve the app_id.
  
    Note that if the app_id is specified, the internal appid must be used;
    this may include a partition and a domain. It is often easier to let
    remote_api_stub retreive the app_id automatically.
  
  
    Args:
      app_id: The app_id of your app, as declared in app.yaml, or None.
      path: The path to the remote_api handler for your app
        (for example, '/_ah/remote_api').
      auth_func: A function that takes no arguments and returns a
        (username, password) tuple. This will be called if your application
        requires authentication to access the remote_api handler (it should!)
        and you do not already have a valid auth cookie.
      servername: The hostname your app is deployed on. Defaults to
        <app_id>.appspot.com.
      rpc_server_factory: A factory to construct the rpc server for the datastore.
      rtok: The validation token to sent with app_id lookups. If None, a random
        token is used.
      secure: Use SSL when communicating with the server.
      services: A list of services to set up stubs for. If specified, only those
        services are configured; by default all supported services are configured.
      default_auth_domain: The authentication domain to use by default.
      save_cookies: Forwarded to rpc_server_factory function.
      use_remote_datastore: Whether to use RemoteDatastoreStub instead of passing
        through datastore requests. RemoteDatastoreStub batches transactional
        datastore requests since, in production, datastore requires are scoped to
        a single request.
      use_async_rpc: A boolean indicating whether or not to make RPC calls in a
        separate thread.
  
    Returns:
      server, the server created by rpc_server_factory, which may be useful for
        calling the application directly.
  
    Raises:
      urllib2.HTTPError: if app_id is not provided and there is an error while
        retrieving it.
      ConfigurationError: if there is a error configuring the DatstoreFileStub.
    """
    if not servername and not app_id:
      raise ConfigurationError('app_id or servername required')
    if not servername:
      servername = '%s.appspot.com' % (app_id,)
    server = rpc_server_factory(servername, auth_func, GetUserAgent(),
                                GetSourceName(), save_cookies=save_cookies,
                                debug_data=False, secure=secure)
    if not app_id:
      app_id = GetRemoteAppIdFromServer(server, path, rtok)
  
    ConfigureRemoteApiFromServer(server, path, app_id, services,
                                 default_auth_domain, use_remote_datastore,
                                 use_async_rpc)
    return server
  
  
  def MaybeInvokeAuthentication():
    """Sends an empty request through to the configured end-point.
  
    If authentication is necessary, this will cause the rpc_server to invoke
    interactive authentication.
    """
    datastore_stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3')
    if isinstance(datastore_stub, RemoteStub):
      datastore_stub._server.Send(datastore_stub._path, payload=None)
    else:
      raise ConfigurationError('remote_api is not configured.')
  
  
  
  ConfigureRemoteDatastore = ConfigureRemoteApi
  
files: AppServer/google/appengine/ext/remote_api/remote_api_stub.py
files: #!/usr/bin/env python
  #
  # Copyright 2007 Google Inc.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  #
  """The main entry point for the new development server."""
  
  
  import argparse
  import errno
  import getpass
  import itertools
  import logging
  import os
  import sys
  import tempfile
  import time
  
  from google.appengine.datastore import datastore_stub_util
  from google.appengine.tools import boolean_action
  from google.appengine.tools.devappserver2.admin import admin_server
  from google.appengine.tools.devappserver2 import api_server
  from google.appengine.tools.devappserver2 import application_configuration
  from google.appengine.tools.devappserver2 import dispatcher
  from google.appengine.tools.devappserver2 import login
  from google.appengine.tools.devappserver2 import runtime_config_pb2
  from google.appengine.tools.devappserver2 import shutdown
  from google.appengine.tools.devappserver2 import update_checker
  from google.appengine.tools.devappserver2 import wsgi_request_info
  
  from appscale.common import appscale_info
  
  # Initialize logging early -- otherwise some library packages may
  # pre-empt our log formatting.  NOTE: the level is provisional; it may
  # be changed in main() based on the --debug flag.
  logging.basicConfig(
      level=logging.INFO,
      format='%(levelname)-8s %(asctime)s %(filename)s:%(lineno)s] %(message)s')
  
  # Valid choices for --log_level and their corresponding constants in
  # runtime_config_pb2.Config.stderr_log_level.
  _LOG_LEVEL_TO_RUNTIME_CONSTANT = {
      'debug': 0,
      'info': 1,
      'warning': 2,
      'error': 3,
      'critical': 4,
  }
  
  # Valid choices for --dev_appserver_log_level and their corresponding Python
  # logging levels
  _LOG_LEVEL_TO_PYTHON_CONSTANT = {
      'debug': logging.DEBUG,
      'info': logging.INFO,
      'warning': logging.WARNING,
      'error': logging.ERROR,
      'critical': logging.CRITICAL,
  }
  
  
  def _generate_storage_paths(app_id):
    """Yield an infinite sequence of possible storage paths."""
    if sys.platform == 'win32':
      # The temp directory is per-user on Windows so there is no reason to add
      # the username to the generated directory name.
      user_format = ''
    else:
      try:
        user_name = getpass.getuser()
      except Exception:  # The possible set of exceptions is not documented.
        user_format = ''
      else:
        user_format = '.%s' % user_name
  
    tempdir = tempfile.gettempdir()
    yield os.path.join(tempdir, 'appengine.%s%s' % (app_id, user_format))
    for i in itertools.count(1):
      yield os.path.join(tempdir, 'appengine.%s%s.%d' % (app_id, user_format, i))
  
  
  def _get_storage_path(path, app_id):
    """Returns a path to the directory where stub data can be stored."""
    _, _, app_id = app_id.replace(':', '_').rpartition('~')
    if path is None:
      for path in _generate_storage_paths(app_id):
        try:
          os.mkdir(path, 0700)
        except OSError, e:
          if e.errno == errno.EEXIST:
            # Check that the directory is only accessable by the current user to
            # protect against an attacker creating the directory in advance in
            # order to access any created files. Windows has per-user temporary
            # directories and st_mode does not include per-user permission
            # information so assume that it is safe.
            if sys.platform == 'win32' or (
                (os.stat(path).st_mode & 0777) == 0700 and os.path.isdir(path)):
              return path
            else:
              continue
          raise
        else:
          return path
    elif not os.path.exists(path):
      os.mkdir(path)
      return path
    elif not os.path.isdir(path):
      raise IOError('the given storage path %r is a file, a directory was '
                    'expected' % path)
    else:
      return path
  
  
  class PortParser(object):
    """A parser for ints that represent ports."""
  
    def __init__(self, allow_port_zero=True):
      self._min_port = 0 if allow_port_zero else 1
  
    def __call__(self, value):
      try:
        port = int(value)
      except ValueError:
        raise argparse.ArgumentTypeError('Invalid port: %r' % value)
      if port < self._min_port or port >= (1 << 16):
        raise argparse.ArgumentTypeError('Invalid port: %d' % port)
      return port
  
  
  def parse_max_module_instances(value):
    """Returns the parsed value for the --max_module_instances flag.
  
    Args:
      value: A str containing the flag value for parse. The format should follow
          one of the following examples:
            1. "5" - All modules are limited to 5 instances.
            2. "default:3,backend:20" - The default module can have 3 instances,
               "backend" can have 20 instances and all other modules are
                unaffected.
    """
    if ':' not in value:
      try:
        max_module_instances = int(value)
      except ValueError:
        raise argparse.ArgumentTypeError('Invalid instance count: %r' % value)
      else:
        if not max_module_instances:
          raise argparse.ArgumentTypeError(
              'Cannot specify zero instances for all modules')
        return max_module_instances
    else:
      module_to_max_instances = {}
      for module_instance_max in value.split(','):
        try:
          module_name, max_instances = module_instance_max.split(':')
          max_instances = int(max_instances)
        except ValueError:
          raise argparse.ArgumentTypeError(
              'Expected "module:max_instances": %r' % module_instance_max)
        else:
          module_name = module_name.strip()
          if module_name in module_to_max_instances:
            raise argparse.ArgumentTypeError(
                'Duplicate max instance value: %r' % module_name)
          module_to_max_instances[module_name] = max_instances
      return module_to_max_instances
  
  
  def create_command_line_parser():
    """Returns an argparse.ArgumentParser to parse command line arguments."""
    # TODO: Add more robust argument validation. Consider what flags
    # are actually needed.
  
    parser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('yaml_files', nargs='+')
  
    common_group = parser.add_argument_group('Common')
    common_group.add_argument(
        '--host', default='localhost',
        help='host name to which application modules should bind')
    common_group.add_argument(
        '--port', type=PortParser(), default=8080,
        help='lowest port to which application modules should bind')
    common_group.add_argument(
        '--admin_host', default='localhost',
        help='host name to which the admin server should bind')
    common_group.add_argument(
        '--admin_port', type=PortParser(), default=8000,
        help='port to which the admin server should bind')
    common_group.add_argument(
        '--auth_domain', default='gmail.com',
        help='name of the authorization domain to use')
    common_group.add_argument(
        '--storage_path', metavar='PATH',
        type=os.path.expanduser,
        help='path to the data (datastore, blobstore, etc.) associated with the '
        'application.')
    common_group.add_argument(
        '--log_level', default='info',
        choices=_LOG_LEVEL_TO_RUNTIME_CONSTANT.keys(),
        help='the log level below which logging messages generated by '
        'application code will not be displayed on the console')
    common_group.add_argument(
        '--max_module_instances',
        type=parse_max_module_instances,
        help='the maximum number of runtime instances that can be started for a '
        'particular module - the value can be an integer, in what case all '
        'modules are limited to that number of instances or a comma-seperated '
        'list of module:max_instances e.g. "default:5,backend:3"')
    common_group.add_argument(
        '--use_mtime_file_watcher',
        action=boolean_action.BooleanAction,
        const=True,
        default=False,
        help='use mtime polling for detecting source code changes - useful if '
        'modifying code from a remote machine using a distributed file system')
  
    # PHP
    php_group = parser.add_argument_group('PHP')
    php_group.add_argument('--php_executable_path', metavar='PATH',
                           help='path to the PHP executable',
                           default='php-cgi')
    php_group.add_argument('--php_remote_debugging',
                           action=boolean_action.BooleanAction,
                           const=True,
                           default=False,
                           help='enable XDebug remote debugging')
  
    # Python
    python_group = parser.add_argument_group('Python')
    python_group.add_argument(
        '--python_startup_script',
        help='the script to run at the startup of new Python runtime instances '
        '(useful for tools such as debuggers.')
    python_group.add_argument(
        '--python_startup_args',
        help='the arguments made available to the script specified in '
        '--python_startup_script.')
  
    # Blobstore
    blobstore_group = parser.add_argument_group('Blobstore API')
    blobstore_group.add_argument(
        '--blobstore_path',
        type=os.path.expanduser,
        help='path to directory used to store blob contents '
        '(defaults to a subdirectory of --storage_path if not set)',
        default=None)
  
    # Cloud SQL
    cloud_sql_group = parser.add_argument_group('Cloud SQL')
    cloud_sql_group.add_argument(
        '--mysql_host',
        default='localhost',
        help='host name of a running MySQL server used for simulated Google '
        'Cloud SQL storage')
    cloud_sql_group.add_argument(
        '--mysql_port', type=PortParser(allow_port_zero=False),
        default=3306,
        help='port number of a running MySQL server used for simulated Google '
        'Cloud SQL storage')
    cloud_sql_group.add_argument(
        '--mysql_user',
        default='',
        help='username to use when connecting to the MySQL server specified in '
        '--mysql_host and --mysql_port or --mysql_socket')
    cloud_sql_group.add_argument(
        '--mysql_password',
        default='',
        help='passpord to use when connecting to the MySQL server specified in '
        '--mysql_host and --mysql_port or --mysql_socket')
    cloud_sql_group.add_argument(
        '--mysql_socket',
        help='path to a Unix socket file to use when connecting to a running '
        'MySQL server used for simulated Google Cloud SQL storage')
  
    # Datastore
    datastore_group = parser.add_argument_group('Datastore API')
    datastore_group.add_argument(
        '--datastore_path',
        type=os.path.expanduser,
        default=None,
        help='path to a file used to store datastore contents '
        '(defaults to a file in --storage_path if not set)',)
    datastore_group.add_argument('--clear_datastore',
                                 action=boolean_action.BooleanAction,
                                 const=True,
                                 default=False,
                                 help='clear the datastore on startup')
    datastore_group.add_argument(
        '--datastore_consistency_policy',
        default='time',
        choices=['consistent', 'random', 'time'],
        help='the policy to apply when deciding whether a datastore write should '
        'appear in global queries')
    datastore_group.add_argument(
        '--require_indexes',
        action=boolean_action.BooleanAction,
        const=True,
        default=False,
        help='generate an error on datastore queries that '
        'requires a composite index not found in index.yaml')
    datastore_group.add_argument(
        '--auto_id_policy',
        default=datastore_stub_util.SCATTERED,
        choices=[datastore_stub_util.SEQUENTIAL,
                 datastore_stub_util.SCATTERED],
        help='the type of sequence from which the datastore stub '
        'assigns automatic IDs. NOTE: Sequential IDs are '
        'deprecated. This flag will be removed in a future '
        'release. Please do not rely on sequential IDs in your '
        'tests.')
  
    # Logs
    logs_group = parser.add_argument_group('Logs API')
    logs_group.add_argument(
        '--logs_path', default=None,
        help='path to a file used to store request logs (defaults to a file in '
        '--storage_path if not set)',)
  
    # Mail
    mail_group = parser.add_argument_group('Mail API')
    mail_group.add_argument(
        '--show_mail_body',
        action=boolean_action.BooleanAction,
        const=True,
        default=False,
        help='logs the contents of e-mails sent using the Mail API')
    mail_group.add_argument(
        '--enable_sendmail',
        action=boolean_action.BooleanAction,
        const=True,
        default=False,
        help='use the "sendmail" tool to transmit e-mail sent '
        'using the Mail API (ignored if --smpt_host is set)')
    mail_group.add_argument(
        '--smtp_host', default='',
        help='host name of an SMTP server to use to transmit '
        'e-mail sent using the Mail API')
    mail_group.add_argument(
        '--smtp_port', default=25,
        type=PortParser(allow_port_zero=False),
        help='port number of an SMTP server to use to transmit '
        'e-mail sent using the Mail API (ignored if --smtp_host '
        'is not set)')
    mail_group.add_argument(
        '--smtp_user', default='',
        help='username to use when connecting to the SMTP server '
        'specified in --smtp_host and --smtp_port')
    mail_group.add_argument(
        '--smtp_password', default='',
        help='password to use when connecting to the SMTP server '
        'specified in --smtp_host and --smtp_port')
  
    # Matcher
    prospective_search_group = parser.add_argument_group('Prospective Search API')
    prospective_search_group.add_argument(
        '--prospective_search_path', default=None,
        type=os.path.expanduser,
        help='path to a file used to store the prospective '
        'search subscription index (defaults to a file in '
        '--storage_path if not set)')
    prospective_search_group.add_argument(
        '--clear_prospective_search',
        action=boolean_action.BooleanAction,
        const=True,
        default=False,
        help='clear the prospective search subscription index')
  
    # Search
    search_group = parser.add_argument_group('Search API')
    search_group.add_argument(
        '--search_indexes_path', default=None,
        type=os.path.expanduser,
        help='path to a file used to store search indexes '
        '(defaults to a file in --storage_path if not set)',)
    search_group.add_argument(
        '--clear_search_indexes',
        action=boolean_action.BooleanAction,
        const=True,
        default=False,
        help='clear the search indexes')
  
    # Taskqueue
    taskqueue_group = parser.add_argument_group('Task Queue API')
    taskqueue_group.add_argument(
        '--enable_task_running',
        action=boolean_action.BooleanAction,
        const=True,
        default=True,
        help='run "push" tasks created using the taskqueue API automatically')
  
    # Misc
    misc_group = parser.add_argument_group('Miscellaneous')
    misc_group.add_argument(
        '--allow_skipped_files',
        action=boolean_action.BooleanAction,
        const=True,
        default=False,
        help='make files specified in the app.yaml "skip_files" or "static"'
        'handles readable by the application.')
    misc_group.add_argument(
        '--api_port', type=PortParser(), default=0,
        help='port to which the server for API calls should bind')
    misc_group.add_argument(
        '--automatic_restart',
        action=boolean_action.BooleanAction,
        const=True,
        default=True,
        help=('restart instances automatically when files relevant to their '
              'module are changed'))
    misc_group.add_argument(
        '--dev_appserver_log_level', default='info',
        choices=_LOG_LEVEL_TO_PYTHON_CONSTANT.keys(),
        help='the log level below which logging messages generated by '
        'the development server will not be displayed on the console (this '
        'flag is more useful for diagnosing problems in dev_appserver.py rather '
        'than in application code)')
    misc_group.add_argument(
        '--skip_sdk_update_check',
        action=boolean_action.BooleanAction,
        const=True,
        default=False,
        help='skip checking for SDK updates (if false, use .appcfg_nag to '
        'decide)')
  
    # AppScale
    appscale_group = parser.add_argument_group('AppScale')
    appscale_group.add_argument(
      '--external_api_port', type=int,
      help='The port of the external server that handles API calls')
    appscale_group.add_argument(
      '--login_server',
      help='the FQDN or IP address where users should be redirected to when the '
      'app needs them to log in on a given URL.')
    appscale_group.add_argument(
      '--nginx_host',
      help='the FQDN or IP address where Task Queue tasks should sent to, so '
      'that they are evenly distributed amongst AppServers.')
    appscale_group.add_argument(
      '--xmpp_path',
      help='the FQDN or IP address where ejabberd is running, so that we know '
      'where XMPP connections should be made to.')
    appscale_group.add_argument(
      '--uaserver_path',
      help='the FQDN or IP address where the UserAppServer runs.')
    appscale_group.add_argument(
      '--trusted',
      action=boolean_action.BooleanAction,
      const=True,
      default=False,
      help='if this application can read data stored by other applications.')
    appscale_group.add_argument('--pidfile', help='create pidfile at location')
  
    return parser
  
  PARSER = create_command_line_parser()
  
  
  def _clear_datastore_storage(datastore_path):
    """Delete the datastore storage file at the given path."""
    # lexists() returns True for broken symlinks, where exists() returns False.
    if os.path.lexists(datastore_path):
      try:
        os.remove(datastore_path)
      except OSError, e:
        logging.warning('Failed to remove datastore file %r: %s',
                        datastore_path,
                        e)
  
  
  def _clear_prospective_search_storage(prospective_search_path):
    """Delete the perspective search storage file at the given path."""
    # lexists() returns True for broken symlinks, where exists() returns False.
    if os.path.lexists(prospective_search_path):
      try:
        os.remove(prospective_search_path)
      except OSError, e:
        logging.warning('Failed to remove prospective search file %r: %s',
                        prospective_search_path,
                        e)
  
  
  def _clear_search_indexes_storage(search_index_path):
    """Delete the search indexes storage file at the given path."""
    # lexists() returns True for broken symlinks, where exists() returns False.
    if os.path.lexists(search_index_path):
      try:
        os.remove(search_index_path)
      except OSError, e:
        logging.warning('Failed to remove search indexes file %r: %s',
                        search_index_path,
                        e)
  
  
  def _setup_environ(app_id, version_id):
    """Sets up the os.environ dictionary for the front-end server and API server.
  
    This function should only be called once.
  
    Args:
      app_id: The id of the application.
    """
    os.environ['APPLICATION_ID'] = app_id
    # AS: set CURRENT_VERSION_ID so taskqueue can get running version.module.
    os.environ['CURRENT_VERSION_ID'] = version_id
  
  
  class DevelopmentServer(object):
    """Encapsulates the logic for the development server.
  
    Only a single instance of the class may be created per process. See
    _setup_environ.
    """
  
    def __init__(self):
      # A list of servers that are currently running.
      self._running_modules = []
      self._module_to_port = {}
  
    def module_to_address(self, module_name, instance=None):
      """Returns the address of a module."""
      if module_name is None:
        return self._dispatcher.dispatch_address
      return self._dispatcher.get_hostname(
          module_name,
          self._dispatcher.get_default_version(module_name),
          instance)
  
    def start(self, options):
      """Start devappserver2 servers based on the provided command line arguments.
  
      Args:
        options: An argparse.Namespace containing the command line arguments.
      """
      logging.getLogger().setLevel(
          _LOG_LEVEL_TO_PYTHON_CONSTANT[options.dev_appserver_log_level])
  
      configuration = application_configuration.ApplicationConfiguration(
          options.yaml_files)
  
      if options.skip_sdk_update_check:
        logging.info('Skipping SDK update check.')
      else:
        update_checker.check_for_updates(configuration)
  
      if options.port == 0:
        logging.warn('DEFAULT_VERSION_HOSTNAME will not be set correctly with '
                     '--port=0')
  
      _setup_environ(configuration.app_id, configuration.modules[0].version_id)
  
      python_config = runtime_config_pb2.PythonConfig()
      if options.python_startup_script:
        python_config.startup_script = os.path.abspath(
            options.python_startup_script)
        if options.python_startup_args:
          python_config.startup_args = options.python_startup_args
  
      cloud_sql_config = runtime_config_pb2.CloudSQL()
      cloud_sql_config.mysql_host = options.mysql_host
      cloud_sql_config.mysql_port = options.mysql_port
      cloud_sql_config.mysql_user = options.mysql_user
      cloud_sql_config.mysql_password = options.mysql_password
      if options.mysql_socket:
        cloud_sql_config.mysql_socket = options.mysql_socket
  
      if options.max_module_instances is None:
        module_to_max_instances = {}
      elif isinstance(options.max_module_instances, int):
        module_to_max_instances = {
            module_configuration.module_name: options.max_module_instances
            for module_configuration in configuration.modules}
      else:
        module_to_max_instances = options.max_module_instances
  
      self._dispatcher = dispatcher.Dispatcher(
          configuration,
          options.host,
          options.port,
          options.auth_domain,
          _LOG_LEVEL_TO_RUNTIME_CONSTANT[options.log_level],
          options.php_executable_path,
          options.php_remote_debugging,
          python_config,
          cloud_sql_config,
          module_to_max_instances,
          options.use_mtime_file_watcher,
          options.automatic_restart,
          options.allow_skipped_files,
          options.external_api_port)
      request_data = wsgi_request_info.WSGIRequestInfo(self._dispatcher)
  
      storage_path = _get_storage_path(options.storage_path, configuration.app_id)
      datastore_path = options.datastore_path or os.path.join(storage_path,
                                                              'datastore.db')
      logs_path = options.logs_path or os.path.join(storage_path, 'logs.db')
      xsrf_path = os.path.join(storage_path, 'xsrf')
  
      search_index_path = options.search_indexes_path or os.path.join(
          storage_path, 'search_indexes')
  
      prospective_search_path = options.prospective_search_path or os.path.join(
          storage_path, 'prospective-search')
  
      blobstore_path = options.blobstore_path or os.path.join(storage_path,
                                                              'blobs')
  
      if options.clear_datastore:
        _clear_datastore_storage(datastore_path)
  
      if options.clear_prospective_search:
        _clear_prospective_search_storage(prospective_search_path)
  
      if options.clear_search_indexes:
        _clear_search_indexes_storage(search_index_path)
  
      if options.auto_id_policy==datastore_stub_util.SEQUENTIAL:
        logging.warn("--auto_id_policy='sequential' is deprecated. This option "
                     "will be removed in a future release.")
  
      application_address = '%s' % options.host
      if options.port and options.port != 80:
        application_address += ':' + str(options.port)
  
      user_login_url = '/%s?%s=%%s' % (login.LOGIN_URL_RELATIVE,
                                       login.CONTINUE_PARAM)
      user_logout_url = '%s&%s=%s' % (user_login_url, login.ACTION_PARAM,
                                      login.LOGOUT_ACTION)
  
      if options.datastore_consistency_policy == 'time':
        consistency = datastore_stub_util.TimeBasedHRConsistencyPolicy()
      elif options.datastore_consistency_policy == 'random':
        consistency = datastore_stub_util.PseudoRandomHRConsistencyPolicy()
      elif options.datastore_consistency_policy == 'consistent':
        consistency = datastore_stub_util.PseudoRandomHRConsistencyPolicy(1.0)
      else:
        assert 0, ('unknown consistency policy: %r' %
                   options.datastore_consistency_policy)
  
      api_server.maybe_convert_datastore_file_stub_data_to_sqlite(
          configuration.app_id, datastore_path)
      api_server.setup_stubs(
          request_data=request_data,
          app_id=configuration.app_id,
          application_root=configuration.modules[0].application_root,
          # The "trusted" flag is only relevant for Google administrative
          # applications.
          trusted=getattr(options, 'trusted', False),
          blobstore_path=blobstore_path,
          datastore_path=datastore_path,
          datastore_consistency=consistency,
          datastore_require_indexes=options.require_indexes,
          datastore_auto_id_policy=options.auto_id_policy,
          images_host_prefix='http://%s' % application_address,
          logs_path=logs_path,
          mail_smtp_host=options.smtp_host,
          mail_smtp_port=options.smtp_port,
          mail_smtp_user=options.smtp_user,
          mail_smtp_password=options.smtp_password,
          mail_enable_sendmail=options.enable_sendmail,
          mail_show_mail_body=options.show_mail_body,
          matcher_prospective_search_path=prospective_search_path,
          search_index_path=search_index_path,
          taskqueue_auto_run_tasks=options.enable_task_running,
          taskqueue_default_http_server=application_address,
          uaserver_path=options.uaserver_path,
          user_login_url=user_login_url,
          user_logout_url=user_logout_url,
          xmpp_path=options.xmpp_path)
  
      # The APIServer must bind to localhost because that is what the runtime
      # instances talk to.
      apis = api_server.APIServer('localhost', options.api_port,
                                  configuration.app_id)
      apis.start()
      self._running_modules.append(apis)
  
      self._running_modules.append(self._dispatcher)
      self._dispatcher.start(apis.port, request_data)
  
      admin = admin_server.AdminServer(options.admin_host, options.admin_port,
                                       self._dispatcher, configuration, xsrf_path)
      admin.start()
      self._running_modules.append(admin)
  
    def stop(self):
      """Stops all running devappserver2 modules."""
      while self._running_modules:
        self._running_modules.pop().quit()
  
  
  def main():
    shutdown.install_signal_handlers()
    # The timezone must be set in the devappserver2 process rather than just in
    # the runtime so printed log timestamps are consistent and the taskqueue stub
    # expects the timezone to be UTC. The runtime inherits the environment.
    os.environ['TZ'] = 'UTC'
    if hasattr(time, 'tzset'):
      # time.tzet() should be called on Unix, but doesn't exist on Windows.
      time.tzset()
    options = PARSER.parse_args()
    os.environ['MY_IP_ADDRESS'] = options.host
    os.environ['MY_PORT'] = str(options.port)
    os.environ['COOKIE_SECRET'] = appscale_info.get_secret()
    os.environ['NGINX_HOST'] = options.nginx_host
  
    if options.pidfile:
      with open(options.pidfile, 'w') as pidfile:
        pidfile.write(str(os.getpid()))
  
    dev_server = DevelopmentServer()
    try:
      dev_server.start(options)
      shutdown.wait_until_shutdown()
    finally:
      dev_server.stop()
  
  
  if __name__ == '__main__':
    main()
  
files: AppServer/google/appengine/tools/devappserver2/devappserver2.py
files: #!/usr/bin/env python
  #
  # Copyright 2007 Google Inc.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  #
  """The main entry point for the new development server."""
  
  
  import argparse
  import errno
  import getpass
  import itertools
  import logging
  import os
  import sys
  import tempfile
  import time
  
  from google.appengine.datastore import datastore_stub_util
  from google.appengine.tools import boolean_action
  from google.appengine.tools.devappserver2.admin import admin_server
  from google.appengine.tools.devappserver2 import api_server
  from google.appengine.tools.devappserver2 import application_configuration
  from google.appengine.tools.devappserver2 import dispatcher
  from google.appengine.tools.devappserver2 import login
  from google.appengine.tools.devappserver2 import runtime_config_pb2
  from google.appengine.tools.devappserver2 import shutdown
  from google.appengine.tools.devappserver2 import update_checker
  from google.appengine.tools.devappserver2 import wsgi_request_info
  
  from appscale.common import appscale_info
  
  # Initialize logging early -- otherwise some library packages may
  # pre-empt our log formatting.  NOTE: the level is provisional; it may
  # be changed in main() based on the --debug flag.
  logging.basicConfig(
      level=logging.INFO,
      format='%(levelname)-8s %(asctime)s %(filename)s:%(lineno)s] %(message)s')
  
  # Valid choices for --log_level and their corresponding constants in
  # runtime_config_pb2.Config.stderr_log_level.
  _LOG_LEVEL_TO_RUNTIME_CONSTANT = {
      'debug': 0,
      'info': 1,
      'warning': 2,
      'error': 3,
      'critical': 4,
  }
  
  # Valid choices for --dev_appserver_log_level and their corresponding Python
  # logging levels
  _LOG_LEVEL_TO_PYTHON_CONSTANT = {
      'debug': logging.DEBUG,
      'info': logging.INFO,
      'warning': logging.WARNING,
      'error': logging.ERROR,
      'critical': logging.CRITICAL,
  }
  
  
  def _generate_storage_paths(app_id):
    """Yield an infinite sequence of possible storage paths."""
    if sys.platform == 'win32':
      # The temp directory is per-user on Windows so there is no reason to add
      # the username to the generated directory name.
      user_format = ''
    else:
      try:
        user_name = getpass.getuser()
      except Exception:  # The possible set of exceptions is not documented.
        user_format = ''
      else:
        user_format = '.%s' % user_name
  
    tempdir = tempfile.gettempdir()
    yield os.path.join(tempdir, 'appengine.%s%s' % (app_id, user_format))
    for i in itertools.count(1):
      yield os.path.join(tempdir, 'appengine.%s%s.%d' % (app_id, user_format, i))
  
  
  def _get_storage_path(path, app_id):
    """Returns a path to the directory where stub data can be stored."""
    _, _, app_id = app_id.replace(':', '_').rpartition('~')
    if path is None:
      for path in _generate_storage_paths(app_id):
        try:
          os.mkdir(path, 0700)
        except OSError, e:
          if e.errno == errno.EEXIST:
            # Check that the directory is only accessable by the current user to
            # protect against an attacker creating the directory in advance in
            # order to access any created files. Windows has per-user temporary
            # directories and st_mode does not include per-user permission
            # information so assume that it is safe.
            if sys.platform == 'win32' or (
                (os.stat(path).st_mode & 0777) == 0700 and os.path.isdir(path)):
              return path
            else:
              continue
          raise
        else:
          return path
    elif not os.path.exists(path):
      os.mkdir(path)
      return path
    elif not os.path.isdir(path):
      raise IOError('the given storage path %r is a file, a directory was '
                    'expected' % path)
    else:
      return path
  
  
  class PortParser(object):
    """A parser for ints that represent ports."""
  
    def __init__(self, allow_port_zero=True):
      self._min_port = 0 if allow_port_zero else 1
  
    def __call__(self, value):
      try:
        port = int(value)
      except ValueError:
        raise argparse.ArgumentTypeError('Invalid port: %r' % value)
      if port < self._min_port or port >= (1 << 16):
        raise argparse.ArgumentTypeError('Invalid port: %d' % port)
      return port
  
  
  def parse_max_module_instances(value):
    """Returns the parsed value for the --max_module_instances flag.
  
    Args:
      value: A str containing the flag value for parse. The format should follow
          one of the following examples:
            1. "5" - All modules are limited to 5 instances.
            2. "default:3,backend:20" - The default module can have 3 instances,
               "backend" can have 20 instances and all other modules are
                unaffected.
    """
    if ':' not in value:
      try:
        max_module_instances = int(value)
      except ValueError:
        raise argparse.ArgumentTypeError('Invalid instance count: %r' % value)
      else:
        if not max_module_instances:
          raise argparse.ArgumentTypeError(
              'Cannot specify zero instances for all modules')
        return max_module_instances
    else:
      module_to_max_instances = {}
      for module_instance_max in value.split(','):
        try:
          module_name, max_instances = module_instance_max.split(':')
          max_instances = int(max_instances)
        except ValueError:
          raise argparse.ArgumentTypeError(
              'Expected "module:max_instances": %r' % module_instance_max)
        else:
          module_name = module_name.strip()
          if module_name in module_to_max_instances:
            raise argparse.ArgumentTypeError(
                'Duplicate max instance value: %r' % module_name)
          module_to_max_instances[module_name] = max_instances
      return module_to_max_instances
  
  
  def create_command_line_parser():
    """Returns an argparse.ArgumentParser to parse command line arguments."""
    # TODO: Add more robust argument validation. Consider what flags
    # are actually needed.
  
    parser = argparse.ArgumentParser(
        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
    parser.add_argument('yaml_files', nargs='+')
  
    common_group = parser.add_argument_group('Common')
    common_group.add_argument(
        '--host', default='localhost',
        help='host name to which application modules should bind')
    common_group.add_argument(
        '--port', type=PortParser(), default=8080,
        help='lowest port to which application modules should bind')
    common_group.add_argument(
        '--admin_host', default='localhost',
        help='host name to which the admin server should bind')
    common_group.add_argument(
        '--admin_port', type=PortParser(), default=8000,
        help='port to which the admin server should bind')
    common_group.add_argument(
        '--auth_domain', default='gmail.com',
        help='name of the authorization domain to use')
    common_group.add_argument(
        '--storage_path', metavar='PATH',
        type=os.path.expanduser,
        help='path to the data (datastore, blobstore, etc.) associated with the '
        'application.')
    common_group.add_argument(
        '--log_level', default='info',
        choices=_LOG_LEVEL_TO_RUNTIME_CONSTANT.keys(),
        help='the log level below which logging messages generated by '
        'application code will not be displayed on the console')
    common_group.add_argument(
        '--max_module_instances',
        type=parse_max_module_instances,
        help='the maximum number of runtime instances that can be started for a '
        'particular module - the value can be an integer, in what case all '
        'modules are limited to that number of instances or a comma-seperated '
        'list of module:max_instances e.g. "default:5,backend:3"')
    common_group.add_argument(
        '--use_mtime_file_watcher',
        action=boolean_action.BooleanAction,
        const=True,
        default=False,
        help='use mtime polling for detecting source code changes - useful if '
        'modifying code from a remote machine using a distributed file system')
  
    # PHP
    php_group = parser.add_argument_group('PHP')
    php_group.add_argument('--php_executable_path', metavar='PATH',
                           help='path to the PHP executable',
                           default='php-cgi')
    php_group.add_argument('--php_remote_debugging',
                           action=boolean_action.BooleanAction,
                           const=True,
                           default=False,
                           help='enable XDebug remote debugging')
  
    # Python
    python_group = parser.add_argument_group('Python')
    python_group.add_argument(
        '--python_startup_script',
        help='the script to run at the startup of new Python runtime instances '
        '(useful for tools such as debuggers.')
    python_group.add_argument(
        '--python_startup_args',
        help='the arguments made available to the script specified in '
        '--python_startup_script.')
  
    # Blobstore
    blobstore_group = parser.add_argument_group('Blobstore API')
    blobstore_group.add_argument(
        '--blobstore_path',
        type=os.path.expanduser,
        help='path to directory used to store blob contents '
        '(defaults to a subdirectory of --storage_path if not set)',
        default=None)
  
    # Cloud SQL
    cloud_sql_group = parser.add_argument_group('Cloud SQL')
    cloud_sql_group.add_argument(
        '--mysql_host',
        default='localhost',
        help='host name of a running MySQL server used for simulated Google '
        'Cloud SQL storage')
    cloud_sql_group.add_argument(
        '--mysql_port', type=PortParser(allow_port_zero=False),
        default=3306,
        help='port number of a running MySQL server used for simulated Google '
        'Cloud SQL storage')
    cloud_sql_group.add_argument(
        '--mysql_user',
        default='',
        help='username to use when connecting to the MySQL server specified in '
        '--mysql_host and --mysql_port or --mysql_socket')
    cloud_sql_group.add_argument(
        '--mysql_password',
        default='',
        help='passpord to use when connecting to the MySQL server specified in '
        '--mysql_host and --mysql_port or --mysql_socket')
    cloud_sql_group.add_argument(
        '--mysql_socket',
        help='path to a Unix socket file to use when connecting to a running '
        'MySQL server used for simulated Google Cloud SQL storage')
  
    # Datastore
    datastore_group = parser.add_argument_group('Datastore API')
    datastore_group.add_argument(
        '--datastore_path',
        type=os.path.expanduser,
        default=None,
        help='path to a file used to store datastore contents '
        '(defaults to a file in --storage_path if not set)',)
    datastore_group.add_argument('--clear_datastore',
                                 action=boolean_action.BooleanAction,
                                 const=True,
                                 default=False,
                                 help='clear the datastore on startup')
    datastore_group.add_argument(
        '--datastore_consistency_policy',
        default='time',
        choices=['consistent', 'random', 'time'],
        help='the policy to apply when deciding whether a datastore write should '
        'appear in global queries')
    datastore_group.add_argument(
        '--require_indexes',
        action=boolean_action.BooleanAction,
        const=True,
        default=False,
        help='generate an error on datastore queries that '
        'requires a composite index not found in index.yaml')
    datastore_group.add_argument(
        '--auto_id_policy',
        default=datastore_stub_util.SCATTERED,
        choices=[datastore_stub_util.SEQUENTIAL,
                 datastore_stub_util.SCATTERED],
        help='the type of sequence from which the datastore stub '
        'assigns automatic IDs. NOTE: Sequential IDs are '
        'deprecated. This flag will be removed in a future '
        'release. Please do not rely on sequential IDs in your '
        'tests.')
  
    # Logs
    logs_group = parser.add_argument_group('Logs API')
    logs_group.add_argument(
        '--logs_path', default=None,
        help='path to a file used to store request logs (defaults to a file in '
        '--storage_path if not set)',)
  
    # Mail
    mail_group = parser.add_argument_group('Mail API')
    mail_group.add_argument(
        '--show_mail_body',
        action=boolean_action.BooleanAction,
        const=True,
        default=False,
        help='logs the contents of e-mails sent using the Mail API')
    mail_group.add_argument(
        '--enable_sendmail',
        action=boolean_action.BooleanAction,
        const=True,
        default=False,
        help='use the "sendmail" tool to transmit e-mail sent '
        'using the Mail API (ignored if --smpt_host is set)')
    mail_group.add_argument(
        '--smtp_host', default='',
        help='host name of an SMTP server to use to transmit '
        'e-mail sent using the Mail API')
    mail_group.add_argument(
        '--smtp_port', default=25,
        type=PortParser(allow_port_zero=False),
        help='port number of an SMTP server to use to transmit '
        'e-mail sent using the Mail API (ignored if --smtp_host '
        'is not set)')
    mail_group.add_argument(
        '--smtp_user', default='',
        help='username to use when connecting to the SMTP server '
        'specified in --smtp_host and --smtp_port')
    mail_group.add_argument(
        '--smtp_password', default='',
        help='password to use when connecting to the SMTP server '
        'specified in --smtp_host and --smtp_port')
  
    # Matcher
    prospective_search_group = parser.add_argument_group('Prospective Search API')
    prospective_search_group.add_argument(
        '--prospective_search_path', default=None,
        type=os.path.expanduser,
        help='path to a file used to store the prospective '
        'search subscription index (defaults to a file in '
        '--storage_path if not set)')
    prospective_search_group.add_argument(
        '--clear_prospective_search',
        action=boolean_action.BooleanAction,
        const=True,
        default=False,
        help='clear the prospective search subscription index')
  
    # Search
    search_group = parser.add_argument_group('Search API')
    search_group.add_argument(
        '--search_indexes_path', default=None,
        type=os.path.expanduser,
        help='path to a file used to store search indexes '
        '(defaults to a file in --storage_path if not set)',)
    search_group.add_argument(
        '--clear_search_indexes',
        action=boolean_action.BooleanAction,
        const=True,
        default=False,
        help='clear the search indexes')
  
    # Taskqueue
    taskqueue_group = parser.add_argument_group('Task Queue API')
    taskqueue_group.add_argument(
        '--enable_task_running',
        action=boolean_action.BooleanAction,
        const=True,
        default=True,
        help='run "push" tasks created using the taskqueue API automatically')
  
    # Misc
    misc_group = parser.add_argument_group('Miscellaneous')
    misc_group.add_argument(
        '--allow_skipped_files',
        action=boolean_action.BooleanAction,
        const=True,
        default=False,
        help='make files specified in the app.yaml "skip_files" or "static"'
        'handles readable by the application.')
    misc_group.add_argument(
        '--api_port', type=PortParser(), default=0,
        help='port to which the server for API calls should bind')
    misc_group.add_argument(
        '--automatic_restart',
        action=boolean_action.BooleanAction,
        const=True,
        default=True,
        help=('restart instances automatically when files relevant to their '
              'module are changed'))
    misc_group.add_argument(
        '--dev_appserver_log_level', default='info',
        choices=_LOG_LEVEL_TO_PYTHON_CONSTANT.keys(),
        help='the log level below which logging messages generated by '
        'the development server will not be displayed on the console (this '
        'flag is more useful for diagnosing problems in dev_appserver.py rather '
        'than in application code)')
    misc_group.add_argument(
        '--skip_sdk_update_check',
        action=boolean_action.BooleanAction,
        const=True,
        default=False,
        help='skip checking for SDK updates (if false, use .appcfg_nag to '
        'decide)')
  
    # AppScale
    appscale_group = parser.add_argument_group('AppScale')
    appscale_group.add_argument(
      '--login_server',
      help='the FQDN or IP address where users should be redirected to when the '
      'app needs them to log in on a given URL.')
    appscale_group.add_argument(
      '--nginx_host',
      help='the FQDN or IP address where Task Queue tasks should sent to, so '
      'that they are evenly distributed amongst AppServers.')
    appscale_group.add_argument(
      '--xmpp_path',
      help='the FQDN or IP address where ejabberd is running, so that we know '
      'where XMPP connections should be made to.')
    appscale_group.add_argument(
      '--uaserver_path',
      help='the FQDN or IP address where the UserAppServer runs.')
    appscale_group.add_argument(
      '--trusted',
      action=boolean_action.BooleanAction,
      const=True,
      default=False,
      help='if this application can read data stored by other applications.')
    appscale_group.add_argument('--pidfile', help='create pidfile at location')
  
    return parser
  
  PARSER = create_command_line_parser()
  
  
  def _clear_datastore_storage(datastore_path):
    """Delete the datastore storage file at the given path."""
    # lexists() returns True for broken symlinks, where exists() returns False.
    if os.path.lexists(datastore_path):
      try:
        os.remove(datastore_path)
      except OSError, e:
        logging.warning('Failed to remove datastore file %r: %s',
                        datastore_path,
                        e)
  
  
  def _clear_prospective_search_storage(prospective_search_path):
    """Delete the perspective search storage file at the given path."""
    # lexists() returns True for broken symlinks, where exists() returns False.
    if os.path.lexists(prospective_search_path):
      try:
        os.remove(prospective_search_path)
      except OSError, e:
        logging.warning('Failed to remove prospective search file %r: %s',
                        prospective_search_path,
                        e)
  
  
  def _clear_search_indexes_storage(search_index_path):
    """Delete the search indexes storage file at the given path."""
    # lexists() returns True for broken symlinks, where exists() returns False.
    if os.path.lexists(search_index_path):
      try:
        os.remove(search_index_path)
      except OSError, e:
        logging.warning('Failed to remove search indexes file %r: %s',
                        search_index_path,
                        e)
  
  
  def _setup_environ(app_id, version_id):
    """Sets up the os.environ dictionary for the front-end server and API server.
  
    This function should only be called once.
  
    Args:
      app_id: The id of the application.
    """
    os.environ['APPLICATION_ID'] = app_id
    # AS: set CURRENT_VERSION_ID so taskqueue can get running version.module.
    os.environ['CURRENT_VERSION_ID'] = version_id
  
  
  class DevelopmentServer(object):
    """Encapsulates the logic for the development server.
  
    Only a single instance of the class may be created per process. See
    _setup_environ.
    """
  
    def __init__(self):
      # A list of servers that are currently running.
      self._running_modules = []
      self._module_to_port = {}
  
    def module_to_address(self, module_name, instance=None):
      """Returns the address of a module."""
      if module_name is None:
        return self._dispatcher.dispatch_address
      return self._dispatcher.get_hostname(
          module_name,
          self._dispatcher.get_default_version(module_name),
          instance)
  
    def start(self, options):
      """Start devappserver2 servers based on the provided command line arguments.
  
      Args:
        options: An argparse.Namespace containing the command line arguments.
      """
      logging.getLogger().setLevel(
          _LOG_LEVEL_TO_PYTHON_CONSTANT[options.dev_appserver_log_level])
  
      configuration = application_configuration.ApplicationConfiguration(
          options.yaml_files)
  
      if options.skip_sdk_update_check:
        logging.info('Skipping SDK update check.')
      else:
        update_checker.check_for_updates(configuration)
  
      if options.port == 0:
        logging.warn('DEFAULT_VERSION_HOSTNAME will not be set correctly with '
                     '--port=0')
  
      _setup_environ(configuration.app_id, configuration.modules[0].version_id)
  
      python_config = runtime_config_pb2.PythonConfig()
      if options.python_startup_script:
        python_config.startup_script = os.path.abspath(
            options.python_startup_script)
        if options.python_startup_args:
          python_config.startup_args = options.python_startup_args
  
      cloud_sql_config = runtime_config_pb2.CloudSQL()
      cloud_sql_config.mysql_host = options.mysql_host
      cloud_sql_config.mysql_port = options.mysql_port
      cloud_sql_config.mysql_user = options.mysql_user
      cloud_sql_config.mysql_password = options.mysql_password
      if options.mysql_socket:
        cloud_sql_config.mysql_socket = options.mysql_socket
  
      if options.max_module_instances is None:
        module_to_max_instances = {}
      elif isinstance(options.max_module_instances, int):
        module_to_max_instances = {
            module_configuration.module_name: options.max_module_instances
            for module_configuration in configuration.modules}
      else:
        module_to_max_instances = options.max_module_instances
  
      self._dispatcher = dispatcher.Dispatcher(
          configuration,
          options.host,
          options.port,
          options.auth_domain,
          _LOG_LEVEL_TO_RUNTIME_CONSTANT[options.log_level],
          options.php_executable_path,
          options.php_remote_debugging,
          python_config,
          cloud_sql_config,
          module_to_max_instances,
          options.use_mtime_file_watcher,
          options.automatic_restart,
          options.allow_skipped_files)
      request_data = wsgi_request_info.WSGIRequestInfo(self._dispatcher)
  
      storage_path = _get_storage_path(options.storage_path, configuration.app_id)
      datastore_path = options.datastore_path or os.path.join(storage_path,
                                                              'datastore.db')
      logs_path = options.logs_path or os.path.join(storage_path, 'logs.db')
      xsrf_path = os.path.join(storage_path, 'xsrf')
  
      search_index_path = options.search_indexes_path or os.path.join(
          storage_path, 'search_indexes')
  
      prospective_search_path = options.prospective_search_path or os.path.join(
          storage_path, 'prospective-search')
  
      blobstore_path = options.blobstore_path or os.path.join(storage_path,
                                                              'blobs')
  
      if options.clear_datastore:
        _clear_datastore_storage(datastore_path)
  
      if options.clear_prospective_search:
        _clear_prospective_search_storage(prospective_search_path)
  
      if options.clear_search_indexes:
        _clear_search_indexes_storage(search_index_path)
  
      if options.auto_id_policy==datastore_stub_util.SEQUENTIAL:
        logging.warn("--auto_id_policy='sequential' is deprecated. This option "
                     "will be removed in a future release.")
  
      application_address = '%s' % options.host
      if options.port and options.port != 80:
        application_address += ':' + str(options.port)
  
      user_login_url = '/%s?%s=%%s' % (login.LOGIN_URL_RELATIVE,
                                       login.CONTINUE_PARAM)
      user_logout_url = '%s&%s=%s' % (user_login_url, login.ACTION_PARAM,
                                      login.LOGOUT_ACTION)
  
      if options.datastore_consistency_policy == 'time':
        consistency = datastore_stub_util.TimeBasedHRConsistencyPolicy()
      elif options.datastore_consistency_policy == 'random':
        consistency = datastore_stub_util.PseudoRandomHRConsistencyPolicy()
      elif options.datastore_consistency_policy == 'consistent':
        consistency = datastore_stub_util.PseudoRandomHRConsistencyPolicy(1.0)
      else:
        assert 0, ('unknown consistency policy: %r' %
                   options.datastore_consistency_policy)
  
      api_server.maybe_convert_datastore_file_stub_data_to_sqlite(
          configuration.app_id, datastore_path)
      api_server.setup_stubs(
          request_data=request_data,
          app_id=configuration.app_id,
          application_root=configuration.modules[0].application_root,
          # The "trusted" flag is only relevant for Google administrative
          # applications.
          trusted=getattr(options, 'trusted', False),
          blobstore_path=blobstore_path,
          datastore_path=datastore_path,
          datastore_consistency=consistency,
          datastore_require_indexes=options.require_indexes,
          datastore_auto_id_policy=options.auto_id_policy,
          images_host_prefix='http://%s' % application_address,
          logs_path=logs_path,
          mail_smtp_host=options.smtp_host,
          mail_smtp_port=options.smtp_port,
          mail_smtp_user=options.smtp_user,
          mail_smtp_password=options.smtp_password,
          mail_enable_sendmail=options.enable_sendmail,
          mail_show_mail_body=options.show_mail_body,
          matcher_prospective_search_path=prospective_search_path,
          search_index_path=search_index_path,
          taskqueue_auto_run_tasks=options.enable_task_running,
          taskqueue_default_http_server=application_address,
          uaserver_path=options.uaserver_path,
          user_login_url=user_login_url,
          user_logout_url=user_logout_url,
          xmpp_path=options.xmpp_path)
  
      # The APIServer must bind to localhost because that is what the runtime
      # instances talk to.
      apis = api_server.APIServer('localhost', options.api_port,
                                  configuration.app_id)
      apis.start()
      self._running_modules.append(apis)
  
      self._running_modules.append(self._dispatcher)
      self._dispatcher.start(apis.port, request_data)
  
      admin = admin_server.AdminServer(options.admin_host, options.admin_port,
                                       self._dispatcher, configuration, xsrf_path)
      admin.start()
      self._running_modules.append(admin)
  
    def stop(self):
      """Stops all running devappserver2 modules."""
      while self._running_modules:
        self._running_modules.pop().quit()
  
  
  def main():
    shutdown.install_signal_handlers()
    # The timezone must be set in the devappserver2 process rather than just in
    # the runtime so printed log timestamps are consistent and the taskqueue stub
    # expects the timezone to be UTC. The runtime inherits the environment.
    os.environ['TZ'] = 'UTC'
    if hasattr(time, 'tzset'):
      # time.tzet() should be called on Unix, but doesn't exist on Windows.
      time.tzset()
    options = PARSER.parse_args()
    os.environ['MY_IP_ADDRESS'] = options.host
    os.environ['MY_PORT'] = str(options.port)
    os.environ['COOKIE_SECRET'] = appscale_info.get_secret()
    os.environ['NGINX_HOST'] = options.nginx_host
  
    if options.pidfile:
      with open(options.pidfile, 'w') as pidfile:
        pidfile.write(str(os.getpid()))
  
    dev_server = DevelopmentServer()
    try:
      dev_server.start(options)
      shutdown.wait_until_shutdown()
    finally:
      dev_server.stop()
  
  
  if __name__ == '__main__':
    main()
  
files: AppServer/google/appengine/tools/devappserver2/devappserver2.py
files: #!/usr/bin/env python
  #
  # Copyright 2007 Google Inc.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  #
  """Manage the lifecycle of modules and dispatch requests to them."""
  
  import collections
  import logging
  import os
  import threading
  import time
  import urlparse
  import wsgiref.headers
  
  from google.appengine.api import request_info
  from google.appengine.tools.devappserver2 import constants
  from google.appengine.tools.devappserver2 import instance
  from google.appengine.tools.devappserver2 import scheduled_executor
  from google.appengine.tools.devappserver2 import module
  from google.appengine.tools.devappserver2 import start_response_utils
  from google.appengine.tools.devappserver2 import thread_executor
  from google.appengine.tools.devappserver2 import wsgi_server
  
  _THREAD_POOL = thread_executor.ThreadExecutor()
  
  ResponseTuple = collections.namedtuple('ResponseTuple',
                                         ['status', 'headers', 'content'])
  
  
  class PortRegistry(object):
    def __init__(self):
      self._ports = {}
      self._ports_lock = threading.RLock()
  
    def add(self, port, _module, inst):
      with self._ports_lock:
        self._ports[port] = (_module, inst)
  
    def get(self, port):
      with self._ports_lock:
        return self._ports[port]
  
  
  class Dispatcher(request_info.Dispatcher):
    """A devappserver2 implementation of request_info.Dispatcher.
  
    In addition to the request_info.Dispatcher interface, it owns modules and
    manages their lifetimes.
    """
  
    def __init__(self,
                 configuration,
                 host,
                 port,
                 auth_domain,
                 runtime_stderr_loglevel,
                 php_executable_path,
                 enable_php_remote_debugging,
                 python_config,
                 cloud_sql_config,
                 module_to_max_instances,
                 use_mtime_file_watcher,
                 automatic_restart,
                 allow_skipped_files,
                 external_api_port=None):
      """Initializer for Dispatcher.
  
      Args:
        configuration: An application_configuration.ApplicationConfiguration
            instance storing the configuration data for the app.
        host: A string containing the host that any HTTP servers should bind to
            e.g. "localhost".
        port: An int specifying the first port where servers should listen.
        auth_domain: A string containing the auth domain to set in the environment
            variables.
        runtime_stderr_loglevel: An int reprenting the minimum logging level at
            which runtime log messages should be written to stderr. See
            devappserver2.py for possible values.
        php_executable_path: A string containing the path to PHP execution e.g.
            "/usr/bin/php-cgi".
        enable_php_remote_debugging: A boolean indicating whether the PHP
            interpreter should be started with XDebug remote debugging enabled.
        python_config: A runtime_config_pb2.PythonConfig instance containing
            Python runtime-specific configuration. If None then defaults are
            used.
        cloud_sql_config: A runtime_config_pb2.CloudSQL instance containing the
            required configuration for local Google Cloud SQL development. If None
            then Cloud SQL will not be available.
        module_to_max_instances: A mapping between a module name and the maximum
            number of instances that can be created (this overrides the settings
            found in the configuration argument) e.g.
            {'default': 10, 'backend': 15}.
        use_mtime_file_watcher: A bool containing whether to use mtime polling to
            monitor file changes even if other options are available on the
            current platform.
        automatic_restart: If True then instances will be restarted when a
            file or configuration change that effects them is detected.
        allow_skipped_files: If True then all files in the application's directory
            are readable, even if they appear in a static handler or "skip_files"
            directive.
        external_api_port: An integer specifying the location of an external API
            server.
      """
      self._configuration = configuration
      self._php_executable_path = php_executable_path
      self._enable_php_remote_debugging = enable_php_remote_debugging
      self._python_config = python_config
      self._cloud_sql_config = cloud_sql_config
      self._request_data = None
      self._api_port = None
      self._external_api_port = external_api_port
      self._running_modules = []
      self._module_configurations = {}
      self._host = host
      self._port = port
      self._auth_domain = auth_domain
      self._runtime_stderr_loglevel = runtime_stderr_loglevel
      self._module_name_to_module = {}
      self._dispatch_server = None
      self._quit_event = threading.Event()  # Set when quit() has been called.
      self._update_checking_thread = threading.Thread(
          target=self._loop_checking_for_updates)
      self._module_to_max_instances = module_to_max_instances or {}
      self._use_mtime_file_watcher = use_mtime_file_watcher
      self._automatic_restart = automatic_restart
      self._allow_skipped_files = allow_skipped_files
      self._executor = scheduled_executor.ScheduledExecutor(_THREAD_POOL)
      self._port_registry = PortRegistry()
  
    def start(self, api_port, request_data):
      """Starts the configured modules.
  
      Args:
        api_port: The port that APIServer listens for RPC requests on.
        request_data: A wsgi_request_info.WSGIRequestInfo that will be provided
            with request information for use by API stubs.
      """
      self._api_port = api_port
      self._request_data = request_data
      port = self._port
      self._executor.start()
      if self._configuration.dispatch:
        self._dispatch_server = wsgi_server.WsgiServer((self._host, port), self)
        self._dispatch_server.start()
        logging.info('Starting dispatcher running at: http://%s:%s', self._host,
                     self._dispatch_server.port)
        self._update_checking_thread.start()
        if port:
          port += 100
        self._port_registry.add(self._dispatch_server.port, None, None)
      for module_configuration in self._configuration.modules:
        self._module_configurations[
            module_configuration.module_name] = module_configuration
        _module, port = self._create_module(module_configuration, port,
                                            self._external_api_port)
        _module.start()
        self._module_name_to_module[module_configuration.module_name] = _module
        logging.info('Starting module "%s" running at: http://%s',
                     module_configuration.module_name, _module.balanced_address)
  
    @property
    def dispatch_port(self):
      """The port that the dispatch HTTP server for the Module is listening on."""
      assert self._dispatch_server, 'dispatch server not running'
      assert self._dispatch_server.ready, 'dispatch server not ready'
      return self._dispatch_server.port
  
    @property
    def host(self):
      """The host that the HTTP server for this Dispatcher is listening on."""
      return self._host
  
    @property
    def dispatch_address(self):
      """The address of the dispatch HTTP server e.g. "localhost:8080"."""
      if self.dispatch_port != 80:
        return '%s:%s' % (self.host, self.dispatch_port)
      else:
        return self.host
  
    def _check_for_updates(self):
      self._configuration.dispatch.check_for_updates()
  
    def _loop_checking_for_updates(self):
      """Loops until the Dispatcher exits, reloading dispatch.yaml config."""
      while not self._quit_event.is_set():
        self._check_for_updates()
        self._quit_event.wait(timeout=1)
  
    def quit(self):
      """Quits all modules."""
      self._executor.quit()
      self._quit_event.set()
      if self._dispatch_server:
        self._dispatch_server.quit()
  
      # AppScale: Prevent instances from serving new requests.
      for _module in self._module_name_to_module.values():
        with _module.graceful_shutdown_lock:
          _module.sigterm_sent = True
  
      logging.info('Waiting for instances to finish serving requests')
      deadline = time.time() + constants.MAX_INSTANCE_RESPONSE_TIME
      while True:
        if time.time() > deadline:
          logging.error('Request timeout while shutting down')
          break
  
        requests_in_progress = False
        for _module in self._module_name_to_module.values():
          with _module.graceful_shutdown_lock:
            if _module.request_count > 0:
              requests_in_progress = True
  
        if not requests_in_progress:
          break
  
        time.sleep(.5)
  
      # End AppScale
  
      for _module in self._module_name_to_module.values():
        _module.quit()
  
    def _create_module(self, module_configuration, port, external_port=None):
      max_instances = self._module_to_max_instances.get(
          module_configuration.module_name)
      module_args = (module_configuration,
                     self._host,
                     port,
                     self._api_port,
                     self._auth_domain,
                     self._runtime_stderr_loglevel,
                     self._php_executable_path,
                     self._enable_php_remote_debugging,
                     self._python_config,
                     self._cloud_sql_config,
                     self._port,
                     self._port_registry,
                     self._request_data,
                     self,
                     max_instances,
                     self._use_mtime_file_watcher,
                     self._automatic_restart,
                     self._allow_skipped_files)
      module_kwargs = {'external_api_port': external_port}
      if module_configuration.manual_scaling:
        _module = module.ManualScalingModule(*module_args, **module_kwargs)
      elif module_configuration.basic_scaling:
        _module = module.BasicScalingModule(*module_args, **module_kwargs)
      else:
        _module = module.AutoScalingModule(*module_args, **module_kwargs)
  
      if port != 0:
        port += 1000
      return _module, port
  
    @property
    def modules(self):
      return self._module_name_to_module.values()
  
    def get_hostname(self, module_name, version, instance_id=None):
      """Returns the hostname for a (module, version, instance_id) tuple.
  
      If instance_id is set, this will return a hostname for that particular
      instances. Otherwise, it will return the hostname for load-balancing.
  
      Args:
        module_name: A str containing the name of the module.
        version: A str containing the version.
        instance_id: An optional str containing the instance ID.
  
      Returns:
        A str containing the hostname.
  
      Raises:
        request_info.ModuleDoesNotExistError: The module does not exist.
        request_info.VersionDoesNotExistError: The version does not exist.
        request_info.InvalidInstanceIdError: The instance ID is not valid for the
            module/version or the module/version uses automatic scaling.
      """
      _module = self._get_module(module_name, version)
      if instance_id is None:
        return _module.balanced_address
      else:
        return _module.get_instance_address(instance_id)
  
    def get_module_names(self):
      """Returns a list of module names."""
      return list(self._module_name_to_module)
  
    def get_module_by_name(self, _module):
      """Returns the module with the given name.
  
      Args:
        _module: A str containing the name of the module.
  
      Returns:
        The module.Module with the provided name.
  
      Raises:
        request_info.ModuleDoesNotExistError: The module does not exist.
      """
      try:
        return self._module_name_to_module[_module]
      except KeyError:
        raise request_info.ModuleDoesNotExistError(_module)
  
    def get_versions(self, _module):
      """Returns a list of versions for a module.
  
      Args:
        _module: A str containing the name of the module.
  
      Returns:
        A list of str containing the versions for the specified module.
  
      Raises:
        request_info.ModuleDoesNotExistError: The module does not exist.
      """
      if _module in self._module_configurations:
        return [self._module_configurations[_module].major_version]
      else:
        raise request_info.ModuleDoesNotExistError(_module)
  
    def get_default_version(self, _module):
      """Returns the default version for a module.
  
      Args:
        _module: A str containing the name of the module.
  
      Returns:
        A str containing the default version for the specified module.
  
      Raises:
        request_info.ModuleDoesNotExistError: The module does not exist.
      """
      if _module in self._module_configurations:
        return self._module_configurations[_module].major_version
      else:
        raise request_info.ModuleDoesNotExistError(_module)
  
    def add_event(self, runnable, eta, service=None, event_id=None):
      """Add a callable to be run at the specified time.
  
      Args:
        runnable: A callable object to call at the specified time.
        eta: An int containing the time to run the event, in seconds since the
            epoch.
        service: A str containing the name of the service that owns this event.
            This should be set if event_id is set.
        event_id: A str containing the id of the event. If set, this can be passed
            to update_event to change the time at which the event should run.
      """
      if service is not None and event_id is not None:
        key = (service, event_id)
      else:
        key = None
      self._executor.add_event(runnable, eta, key)
  
    def update_event(self, eta, service, event_id):
      """Update the eta of a scheduled event.
  
      Args:
        eta: An int containing the time to run the event, in seconds since the
            epoch.
        service: A str containing the name of the service that owns this event.
        event_id: A str containing the id of the event to update.
      """
      self._executor.update_event(eta, (service, event_id))
  
    def _get_module(self, module_name, version):
      if not module_name or module_name not in self._module_name_to_module:
        if 'default' in self._module_name_to_module:
          module_name = 'default'
        elif self._module_name_to_module:
          # If there is no default module, but there are other modules, take any.
          # This is somewhat of a hack, and can be removed if we ever enforce the
          # existence of a default module.
          module_name = self._module_name_to_module.keys()[0]
        else:
          raise request_info.ModuleDoesNotExistError(module_name)
      elif (version is not None and
            version != self._module_configurations[module_name].major_version):
        raise request_info.VersionDoesNotExistError()
      return self._module_name_to_module[module_name]
  
    def set_num_instances(self, module_name, version, num_instances):
      """Sets the number of instances to run for a version of a module.
  
      Args:
        module_name: A str containing the name of the module.
        version: A str containing the version.
        num_instances: An int containing the number of instances to run.
  
      Raises:
        ModuleDoesNotExistError: The module does not exist.
        VersionDoesNotExistError: The version does not exist.
        NotSupportedWithAutoScalingError: The provided module/version uses
            automatic scaling.
      """
      self._get_module(module_name, version).set_num_instances(num_instances)
  
    def get_num_instances(self, module_name, version):
      """Returns the number of instances running for a version of a module.
  
      Returns:
        An int containing the number of instances running for a module version.
  
      Args:
        module_name: A str containing the name of the module.
        version: A str containing the version.
  
      Raises:
        ModuleDoesNotExistError: The module does not exist.
        VersionDoesNotExistError: The version does not exist.
        NotSupportedWithAutoScalingError: The provided module/version uses
            automatic scaling.
      """
      return self._get_module(module_name, version).get_num_instances()
  
    def start_module(self, module_name, version):
      """Starts a module.
  
      Args:
        module_name: A str containing the name of the module.
        version: A str containing the version.
  
      Raises:
        ModuleDoesNotExistError: The module does not exist.
        VersionDoesNotExistError: The version does not exist.
        NotSupportedWithAutoScalingError: The provided module/version uses
            automatic scaling.
      """
      self._get_module(module_name, version).resume()
  
    def stop_module(self, module_name, version):
      """Stops a module.
  
      Args:
        module_name: A str containing the name of the module.
        version: A str containing the version.
  
      Raises:
        ModuleDoesNotExistError: The module does not exist.
        VersionDoesNotExistError: The version does not exist.
        NotSupportedWithAutoScalingError: The provided module/version uses
            automatic scaling.
      """
      self._get_module(module_name, version).suspend()
  
    def send_background_request(self, module_name, version, inst,
                                background_request_id):
      """Dispatch a background thread request.
  
      Args:
        module_name: A str containing the module name to service this
            request.
        version: A str containing the version to service this request.
        inst: The instance to service this request.
        background_request_id: A str containing the unique background thread
            request identifier.
  
      Raises:
        NotSupportedWithAutoScalingError: The provided module/version uses
            automatic scaling.
        BackgroundThreadLimitReachedError: The instance is at its background
            thread capacity.
      """
      _module = self._get_module(module_name, version)
      try:
        inst.reserve_background_thread()
      except instance.CannotAcceptRequests:
        raise request_info.BackgroundThreadLimitReachedError()
      port = _module.get_instance_port(inst.instance_id)
      environ = _module.build_request_environ(
          'GET', '/_ah/background',
          [('X-AppEngine-BackgroundRequest', background_request_id)],
          '', '0.1.0.3', port)
      _THREAD_POOL.submit(self._handle_request,
                          environ,
                          start_response_utils.null_start_response,
                          _module,
                          inst,
                          request_type=instance.BACKGROUND_REQUEST,
                          catch_and_log_exceptions=True)
  
    # TODO: Think of better names for add_async_request and
    # add_request.
    def add_async_request(self, method, relative_url, headers, body, source_ip,
                          module_name=None, version=None, instance_id=None):
      """Dispatch an HTTP request asynchronously.
  
      Args:
        method: A str containing the HTTP method of the request.
        relative_url: A str containing path and query string of the request.
        headers: A list of (key, value) tuples where key and value are both str.
        body: A str containing the request body.
        source_ip: The source ip address for the request.
        module_name: An optional str containing the module name to service this
            request. If unset, the request will be dispatched to the default
            module.
        version: An optional str containing the version to service this request.
            If unset, the request will be dispatched to the default version.
        instance_id: An optional str containing the instance_id of the instance to
            service this request. If unset, the request will be dispatched to
            according to the load-balancing for the module and version.
      """
      if module_name:
        _module = self._get_module(module_name, version)
      else:
        _module = self._module_for_request(urlparse.urlsplit(relative_url).path)
      inst = _module.get_instance(instance_id) if instance_id else None
      port = _module.get_instance_port(instance_id) if instance_id else (
          _module.balanced_port)
      environ = _module.build_request_environ(method, relative_url, headers, body,
                                            source_ip, port)
  
      _THREAD_POOL.submit(self._handle_request,
                          environ,
                          start_response_utils.null_start_response,
                          _module,
                          inst,
                          catch_and_log_exceptions=True)
  
    def add_request(self, method, relative_url, headers, body, source_ip,
                    module_name=None, version=None, instance_id=None,
                    fake_login=False):
      """Process an HTTP request.
  
      Args:
        method: A str containing the HTTP method of the request.
        relative_url: A str containing path and query string of the request.
        headers: A list of (key, value) tuples where key and value are both str.
        body: A str containing the request body.
        source_ip: The source ip address for the request.
        module_name: An optional str containing the module name to service this
            request. If unset, the request will be dispatched according to the
            host header and relative_url.
        version: An optional str containing the version to service this request.
            If unset, the request will be dispatched according to the host header
            and relative_url.
        instance_id: An optional str containing the instance_id of the instance to
            service this request. If unset, the request will be dispatched
            according to the host header and relative_url and, if applicable, the
            load-balancing for the module and version.
        fake_login: A bool indicating whether login checks should be bypassed,
            i.e. "login: required" should be ignored for this request.
  
      Returns:
        A request_info.ResponseTuple containing the response information for the
        HTTP request.
      """
      if module_name:
        _module = self._get_module(module_name, version)
        inst = _module.get_instance(instance_id) if instance_id else None
      else:
        headers_dict = wsgiref.headers.Headers(headers)
        _module, inst = self._resolve_target(
            headers_dict['Host'], urlparse.urlsplit(relative_url).path)
      if inst:
        try:
          port = _module.get_instance_port(inst.instance_id)
        except request_info.NotSupportedWithAutoScalingError:
          port = _module.balanced_port
      else:
        port = _module.balanced_port
      environ = _module.build_request_environ(method, relative_url, headers, body,
                                            source_ip, port,
                                            fake_login=fake_login)
      start_response = start_response_utils.CapturingStartResponse()
      response = self._handle_request(environ,
                                      start_response,
                                      _module,
                                      inst)
      return request_info.ResponseTuple(start_response.status,
                                        start_response.response_headers,
                                        start_response.merged_response(response))
  
    def _resolve_target(self, hostname, path):
      """Returns the module and instance that should handle this request.
  
      Args:
        hostname: A string containing the value of the host header in the request
            or None if one was not present.
        path: A string containing the path of the request.
  
      Returns:
        A tuple (_module, inst) where:
          _module: The module.Module that should handle this request.
          inst: The instance.Instance that should handle this request or None if
              the module's load balancing should decide on the instance.
  
      Raises:
        request_info.ModuleDoesNotExistError: if hostname is not known.
      """
      if self._port == 80:
        default_address = self.host
      else:
        default_address = '%s:%s' % (self.host, self._port)
      if not hostname or hostname == default_address:
        return self._module_for_request(path), None
  
      default_address_offset = hostname.find(default_address)
      if default_address_offset > 0:
        prefix = hostname[:default_address_offset - 1]
        if '.' in prefix:
          raise request_info.ModuleDoesNotExistError(prefix)
        return self._get_module(prefix, None), None
  
      else:
        port = int(os.environ['MY_PORT'])
        try:
          _module, inst = self._port_registry.get(port)
        except KeyError:
          raise request_info.ModuleDoesNotExistError(hostname)
      if not _module:
        _module = self._module_for_request(path)
      return _module, inst
  
    def _handle_request(self, environ, start_response, _module,
                        inst=None, request_type=instance.NORMAL_REQUEST,
                        catch_and_log_exceptions=False):
      """Dispatch a WSGI request.
  
      Args:
        environ: An environ dict for the request as defined in PEP-333.
        start_response: A function with semantics defined in PEP-333.
        _module: The module to dispatch this request to.
        inst: The instance to service this request. If None, the module will
            be left to choose the instance to serve this request.
        request_type: The request_type of this request. See instance.*_REQUEST
            module constants.
        catch_and_log_exceptions: A bool containing whether to catch and log
            exceptions in handling the request instead of leaving it for the
            caller to handle.
  
      Returns:
        An iterable over the response to the request as defined in PEP-333.
      """
      try:
        return _module._handle_request(environ, start_response, inst=inst,
                                     request_type=request_type)
      except:
        if catch_and_log_exceptions:
          logging.exception('Internal error while handling request.')
        else:
          raise
  
    def __call__(self, environ, start_response):
      return self._handle_request(
          environ, start_response, self._module_for_request(environ['PATH_INFO']))
  
    def _module_for_request(self, path):
      dispatch = self._configuration.dispatch
      if dispatch:
        for url, module_name in dispatch.dispatch:
          if (url.path_exact and path == url.path or
              not url.path_exact and path.startswith(url.path)):
            return self._get_module(module_name, None)
      return self._get_module(None, None)
  
files: AppServer/google/appengine/tools/devappserver2/dispatcher.py
files: #!/usr/bin/env python
  #
  # Copyright 2007 Google Inc.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  #
  """Manage the lifecycle of modules and dispatch requests to them."""
  
  import collections
  import logging
  import os
  import threading
  import time
  import urlparse
  import wsgiref.headers
  
  from google.appengine.api import request_info
  from google.appengine.tools.devappserver2 import constants
  from google.appengine.tools.devappserver2 import instance
  from google.appengine.tools.devappserver2 import scheduled_executor
  from google.appengine.tools.devappserver2 import module
  from google.appengine.tools.devappserver2 import start_response_utils
  from google.appengine.tools.devappserver2 import thread_executor
  from google.appengine.tools.devappserver2 import wsgi_server
  
  _THREAD_POOL = thread_executor.ThreadExecutor()
  
  ResponseTuple = collections.namedtuple('ResponseTuple',
                                         ['status', 'headers', 'content'])
  
  
  class PortRegistry(object):
    def __init__(self):
      self._ports = {}
      self._ports_lock = threading.RLock()
  
    def add(self, port, _module, inst):
      with self._ports_lock:
        self._ports[port] = (_module, inst)
  
    def get(self, port):
      with self._ports_lock:
        return self._ports[port]
  
  
  class Dispatcher(request_info.Dispatcher):
    """A devappserver2 implementation of request_info.Dispatcher.
  
    In addition to the request_info.Dispatcher interface, it owns modules and
    manages their lifetimes.
    """
  
    def __init__(self,
                 configuration,
                 host,
                 port,
                 auth_domain,
                 runtime_stderr_loglevel,
                 php_executable_path,
                 enable_php_remote_debugging,
                 python_config,
                 cloud_sql_config,
                 module_to_max_instances,
                 use_mtime_file_watcher,
                 automatic_restart,
                 allow_skipped_files):
      """Initializer for Dispatcher.
  
      Args:
        configuration: An application_configuration.ApplicationConfiguration
            instance storing the configuration data for the app.
        host: A string containing the host that any HTTP servers should bind to
            e.g. "localhost".
        port: An int specifying the first port where servers should listen.
        auth_domain: A string containing the auth domain to set in the environment
            variables.
        runtime_stderr_loglevel: An int reprenting the minimum logging level at
            which runtime log messages should be written to stderr. See
            devappserver2.py for possible values.
        php_executable_path: A string containing the path to PHP execution e.g.
            "/usr/bin/php-cgi".
        enable_php_remote_debugging: A boolean indicating whether the PHP
            interpreter should be started with XDebug remote debugging enabled.
        python_config: A runtime_config_pb2.PythonConfig instance containing
            Python runtime-specific configuration. If None then defaults are
            used.
        cloud_sql_config: A runtime_config_pb2.CloudSQL instance containing the
            required configuration for local Google Cloud SQL development. If None
            then Cloud SQL will not be available.
        module_to_max_instances: A mapping between a module name and the maximum
            number of instances that can be created (this overrides the settings
            found in the configuration argument) e.g.
            {'default': 10, 'backend': 15}.
        use_mtime_file_watcher: A bool containing whether to use mtime polling to
            monitor file changes even if other options are available on the
            current platform.
        automatic_restart: If True then instances will be restarted when a
            file or configuration change that effects them is detected.
        allow_skipped_files: If True then all files in the application's directory
            are readable, even if they appear in a static handler or "skip_files"
            directive.
      """
      self._configuration = configuration
      self._php_executable_path = php_executable_path
      self._enable_php_remote_debugging = enable_php_remote_debugging
      self._python_config = python_config
      self._cloud_sql_config = cloud_sql_config
      self._request_data = None
      self._api_port = None
      self._running_modules = []
      self._module_configurations = {}
      self._host = host
      self._port = port
      self._auth_domain = auth_domain
      self._runtime_stderr_loglevel = runtime_stderr_loglevel
      self._module_name_to_module = {}
      self._dispatch_server = None
      self._quit_event = threading.Event()  # Set when quit() has been called.
      self._update_checking_thread = threading.Thread(
          target=self._loop_checking_for_updates)
      self._module_to_max_instances = module_to_max_instances or {}
      self._use_mtime_file_watcher = use_mtime_file_watcher
      self._automatic_restart = automatic_restart
      self._allow_skipped_files = allow_skipped_files
      self._executor = scheduled_executor.ScheduledExecutor(_THREAD_POOL)
      self._port_registry = PortRegistry()
  
    def start(self, api_port, request_data):
      """Starts the configured modules.
  
      Args:
        api_port: The port that APIServer listens for RPC requests on.
        request_data: A wsgi_request_info.WSGIRequestInfo that will be provided
            with request information for use by API stubs.
      """
      self._api_port = api_port
      self._request_data = request_data
      port = self._port
      self._executor.start()
      if self._configuration.dispatch:
        self._dispatch_server = wsgi_server.WsgiServer((self._host, port), self)
        self._dispatch_server.start()
        logging.info('Starting dispatcher running at: http://%s:%s', self._host,
                     self._dispatch_server.port)
        self._update_checking_thread.start()
        if port:
          port += 100
        self._port_registry.add(self._dispatch_server.port, None, None)
      for module_configuration in self._configuration.modules:
        self._module_configurations[
            module_configuration.module_name] = module_configuration
        _module, port = self._create_module(module_configuration, port)
        _module.start()
        self._module_name_to_module[module_configuration.module_name] = _module
        logging.info('Starting module "%s" running at: http://%s',
                     module_configuration.module_name, _module.balanced_address)
  
    @property
    def dispatch_port(self):
      """The port that the dispatch HTTP server for the Module is listening on."""
      assert self._dispatch_server, 'dispatch server not running'
      assert self._dispatch_server.ready, 'dispatch server not ready'
      return self._dispatch_server.port
  
    @property
    def host(self):
      """The host that the HTTP server for this Dispatcher is listening on."""
      return self._host
  
    @property
    def dispatch_address(self):
      """The address of the dispatch HTTP server e.g. "localhost:8080"."""
      if self.dispatch_port != 80:
        return '%s:%s' % (self.host, self.dispatch_port)
      else:
        return self.host
  
    def _check_for_updates(self):
      self._configuration.dispatch.check_for_updates()
  
    def _loop_checking_for_updates(self):
      """Loops until the Dispatcher exits, reloading dispatch.yaml config."""
      while not self._quit_event.is_set():
        self._check_for_updates()
        self._quit_event.wait(timeout=1)
  
    def quit(self):
      """Quits all modules."""
      self._executor.quit()
      self._quit_event.set()
      if self._dispatch_server:
        self._dispatch_server.quit()
  
      # AppScale: Prevent instances from serving new requests.
      for _module in self._module_name_to_module.values():
        with _module.graceful_shutdown_lock:
          _module.sigterm_sent = True
  
      logging.info('Waiting for instances to finish serving requests')
      deadline = time.time() + constants.MAX_INSTANCE_RESPONSE_TIME
      while True:
        if time.time() > deadline:
          logging.error('Request timeout while shutting down')
          break
  
        requests_in_progress = False
        for _module in self._module_name_to_module.values():
          with _module.graceful_shutdown_lock:
            if _module.request_count > 0:
              requests_in_progress = True
  
        if not requests_in_progress:
          break
  
        time.sleep(.5)
  
      # End AppScale
  
      for _module in self._module_name_to_module.values():
        _module.quit()
  
    def _create_module(self, module_configuration, port):
      max_instances = self._module_to_max_instances.get(
          module_configuration.module_name)
      module_args = (module_configuration,
                     self._host,
                     port,
                     self._api_port,
                     self._auth_domain,
                     self._runtime_stderr_loglevel,
                     self._php_executable_path,
                     self._enable_php_remote_debugging,
                     self._python_config,
                     self._cloud_sql_config,
                     self._port,
                     self._port_registry,
                     self._request_data,
                     self,
                     max_instances,
                     self._use_mtime_file_watcher,
                     self._automatic_restart,
                     self._allow_skipped_files)
      if module_configuration.manual_scaling:
        _module = module.ManualScalingModule(*module_args)
      elif module_configuration.basic_scaling:
        _module = module.BasicScalingModule(*module_args)
      else:
        _module = module.AutoScalingModule(*module_args)
  
      if port != 0:
        port += 1000
      return _module, port
  
    @property
    def modules(self):
      return self._module_name_to_module.values()
  
    def get_hostname(self, module_name, version, instance_id=None):
      """Returns the hostname for a (module, version, instance_id) tuple.
  
      If instance_id is set, this will return a hostname for that particular
      instances. Otherwise, it will return the hostname for load-balancing.
  
      Args:
        module_name: A str containing the name of the module.
        version: A str containing the version.
        instance_id: An optional str containing the instance ID.
  
      Returns:
        A str containing the hostname.
  
      Raises:
        request_info.ModuleDoesNotExistError: The module does not exist.
        request_info.VersionDoesNotExistError: The version does not exist.
        request_info.InvalidInstanceIdError: The instance ID is not valid for the
            module/version or the module/version uses automatic scaling.
      """
      _module = self._get_module(module_name, version)
      if instance_id is None:
        return _module.balanced_address
      else:
        return _module.get_instance_address(instance_id)
  
    def get_module_names(self):
      """Returns a list of module names."""
      return list(self._module_name_to_module)
  
    def get_module_by_name(self, _module):
      """Returns the module with the given name.
  
      Args:
        _module: A str containing the name of the module.
  
      Returns:
        The module.Module with the provided name.
  
      Raises:
        request_info.ModuleDoesNotExistError: The module does not exist.
      """
      try:
        return self._module_name_to_module[_module]
      except KeyError:
        raise request_info.ModuleDoesNotExistError(_module)
  
    def get_versions(self, _module):
      """Returns a list of versions for a module.
  
      Args:
        _module: A str containing the name of the module.
  
      Returns:
        A list of str containing the versions for the specified module.
  
      Raises:
        request_info.ModuleDoesNotExistError: The module does not exist.
      """
      if _module in self._module_configurations:
        return [self._module_configurations[_module].major_version]
      else:
        raise request_info.ModuleDoesNotExistError(_module)
  
    def get_default_version(self, _module):
      """Returns the default version for a module.
  
      Args:
        _module: A str containing the name of the module.
  
      Returns:
        A str containing the default version for the specified module.
  
      Raises:
        request_info.ModuleDoesNotExistError: The module does not exist.
      """
      if _module in self._module_configurations:
        return self._module_configurations[_module].major_version
      else:
        raise request_info.ModuleDoesNotExistError(_module)
  
    def add_event(self, runnable, eta, service=None, event_id=None):
      """Add a callable to be run at the specified time.
  
      Args:
        runnable: A callable object to call at the specified time.
        eta: An int containing the time to run the event, in seconds since the
            epoch.
        service: A str containing the name of the service that owns this event.
            This should be set if event_id is set.
        event_id: A str containing the id of the event. If set, this can be passed
            to update_event to change the time at which the event should run.
      """
      if service is not None and event_id is not None:
        key = (service, event_id)
      else:
        key = None
      self._executor.add_event(runnable, eta, key)
  
    def update_event(self, eta, service, event_id):
      """Update the eta of a scheduled event.
  
      Args:
        eta: An int containing the time to run the event, in seconds since the
            epoch.
        service: A str containing the name of the service that owns this event.
        event_id: A str containing the id of the event to update.
      """
      self._executor.update_event(eta, (service, event_id))
  
    def _get_module(self, module_name, version):
      if not module_name or module_name not in self._module_name_to_module:
        if 'default' in self._module_name_to_module:
          module_name = 'default'
        elif self._module_name_to_module:
          # If there is no default module, but there are other modules, take any.
          # This is somewhat of a hack, and can be removed if we ever enforce the
          # existence of a default module.
          module_name = self._module_name_to_module.keys()[0]
        else:
          raise request_info.ModuleDoesNotExistError(module_name)
      elif (version is not None and
            version != self._module_configurations[module_name].major_version):
        raise request_info.VersionDoesNotExistError()
      return self._module_name_to_module[module_name]
  
    def set_num_instances(self, module_name, version, num_instances):
      """Sets the number of instances to run for a version of a module.
  
      Args:
        module_name: A str containing the name of the module.
        version: A str containing the version.
        num_instances: An int containing the number of instances to run.
  
      Raises:
        ModuleDoesNotExistError: The module does not exist.
        VersionDoesNotExistError: The version does not exist.
        NotSupportedWithAutoScalingError: The provided module/version uses
            automatic scaling.
      """
      self._get_module(module_name, version).set_num_instances(num_instances)
  
    def get_num_instances(self, module_name, version):
      """Returns the number of instances running for a version of a module.
  
      Returns:
        An int containing the number of instances running for a module version.
  
      Args:
        module_name: A str containing the name of the module.
        version: A str containing the version.
  
      Raises:
        ModuleDoesNotExistError: The module does not exist.
        VersionDoesNotExistError: The version does not exist.
        NotSupportedWithAutoScalingError: The provided module/version uses
            automatic scaling.
      """
      return self._get_module(module_name, version).get_num_instances()
  
    def start_module(self, module_name, version):
      """Starts a module.
  
      Args:
        module_name: A str containing the name of the module.
        version: A str containing the version.
  
      Raises:
        ModuleDoesNotExistError: The module does not exist.
        VersionDoesNotExistError: The version does not exist.
        NotSupportedWithAutoScalingError: The provided module/version uses
            automatic scaling.
      """
      self._get_module(module_name, version).resume()
  
    def stop_module(self, module_name, version):
      """Stops a module.
  
      Args:
        module_name: A str containing the name of the module.
        version: A str containing the version.
  
      Raises:
        ModuleDoesNotExistError: The module does not exist.
        VersionDoesNotExistError: The version does not exist.
        NotSupportedWithAutoScalingError: The provided module/version uses
            automatic scaling.
      """
      self._get_module(module_name, version).suspend()
  
    def send_background_request(self, module_name, version, inst,
                                background_request_id):
      """Dispatch a background thread request.
  
      Args:
        module_name: A str containing the module name to service this
            request.
        version: A str containing the version to service this request.
        inst: The instance to service this request.
        background_request_id: A str containing the unique background thread
            request identifier.
  
      Raises:
        NotSupportedWithAutoScalingError: The provided module/version uses
            automatic scaling.
        BackgroundThreadLimitReachedError: The instance is at its background
            thread capacity.
      """
      _module = self._get_module(module_name, version)
      try:
        inst.reserve_background_thread()
      except instance.CannotAcceptRequests:
        raise request_info.BackgroundThreadLimitReachedError()
      port = _module.get_instance_port(inst.instance_id)
      environ = _module.build_request_environ(
          'GET', '/_ah/background',
          [('X-AppEngine-BackgroundRequest', background_request_id)],
          '', '0.1.0.3', port)
      _THREAD_POOL.submit(self._handle_request,
                          environ,
                          start_response_utils.null_start_response,
                          _module,
                          inst,
                          request_type=instance.BACKGROUND_REQUEST,
                          catch_and_log_exceptions=True)
  
    # TODO: Think of better names for add_async_request and
    # add_request.
    def add_async_request(self, method, relative_url, headers, body, source_ip,
                          module_name=None, version=None, instance_id=None):
      """Dispatch an HTTP request asynchronously.
  
      Args:
        method: A str containing the HTTP method of the request.
        relative_url: A str containing path and query string of the request.
        headers: A list of (key, value) tuples where key and value are both str.
        body: A str containing the request body.
        source_ip: The source ip address for the request.
        module_name: An optional str containing the module name to service this
            request. If unset, the request will be dispatched to the default
            module.
        version: An optional str containing the version to service this request.
            If unset, the request will be dispatched to the default version.
        instance_id: An optional str containing the instance_id of the instance to
            service this request. If unset, the request will be dispatched to
            according to the load-balancing for the module and version.
      """
      if module_name:
        _module = self._get_module(module_name, version)
      else:
        _module = self._module_for_request(urlparse.urlsplit(relative_url).path)
      inst = _module.get_instance(instance_id) if instance_id else None
      port = _module.get_instance_port(instance_id) if instance_id else (
          _module.balanced_port)
      environ = _module.build_request_environ(method, relative_url, headers, body,
                                            source_ip, port)
  
      _THREAD_POOL.submit(self._handle_request,
                          environ,
                          start_response_utils.null_start_response,
                          _module,
                          inst,
                          catch_and_log_exceptions=True)
  
    def add_request(self, method, relative_url, headers, body, source_ip,
                    module_name=None, version=None, instance_id=None,
                    fake_login=False):
      """Process an HTTP request.
  
      Args:
        method: A str containing the HTTP method of the request.
        relative_url: A str containing path and query string of the request.
        headers: A list of (key, value) tuples where key and value are both str.
        body: A str containing the request body.
        source_ip: The source ip address for the request.
        module_name: An optional str containing the module name to service this
            request. If unset, the request will be dispatched according to the
            host header and relative_url.
        version: An optional str containing the version to service this request.
            If unset, the request will be dispatched according to the host header
            and relative_url.
        instance_id: An optional str containing the instance_id of the instance to
            service this request. If unset, the request will be dispatched
            according to the host header and relative_url and, if applicable, the
            load-balancing for the module and version.
        fake_login: A bool indicating whether login checks should be bypassed,
            i.e. "login: required" should be ignored for this request.
  
      Returns:
        A request_info.ResponseTuple containing the response information for the
        HTTP request.
      """
      if module_name:
        _module = self._get_module(module_name, version)
        inst = _module.get_instance(instance_id) if instance_id else None
      else:
        headers_dict = wsgiref.headers.Headers(headers)
        _module, inst = self._resolve_target(
            headers_dict['Host'], urlparse.urlsplit(relative_url).path)
      if inst:
        try:
          port = _module.get_instance_port(inst.instance_id)
        except request_info.NotSupportedWithAutoScalingError:
          port = _module.balanced_port
      else:
        port = _module.balanced_port
      environ = _module.build_request_environ(method, relative_url, headers, body,
                                            source_ip, port,
                                            fake_login=fake_login)
      start_response = start_response_utils.CapturingStartResponse()
      response = self._handle_request(environ,
                                      start_response,
                                      _module,
                                      inst)
      return request_info.ResponseTuple(start_response.status,
                                        start_response.response_headers,
                                        start_response.merged_response(response))
  
    def _resolve_target(self, hostname, path):
      """Returns the module and instance that should handle this request.
  
      Args:
        hostname: A string containing the value of the host header in the request
            or None if one was not present.
        path: A string containing the path of the request.
  
      Returns:
        A tuple (_module, inst) where:
          _module: The module.Module that should handle this request.
          inst: The instance.Instance that should handle this request or None if
              the module's load balancing should decide on the instance.
  
      Raises:
        request_info.ModuleDoesNotExistError: if hostname is not known.
      """
      if self._port == 80:
        default_address = self.host
      else:
        default_address = '%s:%s' % (self.host, self._port)
      if not hostname or hostname == default_address:
        return self._module_for_request(path), None
  
      default_address_offset = hostname.find(default_address)
      if default_address_offset > 0:
        prefix = hostname[:default_address_offset - 1]
        if '.' in prefix:
          raise request_info.ModuleDoesNotExistError(prefix)
        return self._get_module(prefix, None), None
  
      else:
        port = int(os.environ['MY_PORT'])
        try:
          _module, inst = self._port_registry.get(port)
        except KeyError:
          raise request_info.ModuleDoesNotExistError(hostname)
      if not _module:
        _module = self._module_for_request(path)
      return _module, inst
  
    def _handle_request(self, environ, start_response, _module,
                        inst=None, request_type=instance.NORMAL_REQUEST,
                        catch_and_log_exceptions=False):
      """Dispatch a WSGI request.
  
      Args:
        environ: An environ dict for the request as defined in PEP-333.
        start_response: A function with semantics defined in PEP-333.
        _module: The module to dispatch this request to.
        inst: The instance to service this request. If None, the module will
            be left to choose the instance to serve this request.
        request_type: The request_type of this request. See instance.*_REQUEST
            module constants.
        catch_and_log_exceptions: A bool containing whether to catch and log
            exceptions in handling the request instead of leaving it for the
            caller to handle.
  
      Returns:
        An iterable over the response to the request as defined in PEP-333.
      """
      try:
        return _module._handle_request(environ, start_response, inst=inst,
                                     request_type=request_type)
      except:
        if catch_and_log_exceptions:
          logging.exception('Internal error while handling request.')
        else:
          raise
  
    def __call__(self, environ, start_response):
      return self._handle_request(
          environ, start_response, self._module_for_request(environ['PATH_INFO']))
  
    def _module_for_request(self, path):
      dispatch = self._configuration.dispatch
      if dispatch:
        for url, module_name in dispatch.dispatch:
          if (url.path_exact and path == url.path or
              not url.path_exact and path.startswith(url.path)):
            return self._get_module(module_name, None)
      return self._get_module(None, None)
  
files: AppServer/google/appengine/tools/devappserver2/dispatcher.py
files: #!/usr/bin/env python
  #
  # Copyright 2007 Google Inc.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  #
  """Manage the lifecycle of runtime processes and dispatch requests to them."""
  
  
  import collections
  import cStringIO
  import functools
  import httplib
  import logging
  import math
  import os
  import os.path
  import random
  import re
  import string
  import struct
  import threading
  import time
  import urllib
  import urlparse
  import wsgiref.headers
  
  from google.appengine.api import api_base_pb
  from google.appengine.api import apiproxy_stub_map
  from google.appengine.api import appinfo
  from google.appengine.api import request_info
  from google.appengine.api.logservice import log_service_pb
  from google.appengine.tools.devappserver2 import application_configuration
  from google.appengine.tools.devappserver2 import blob_image
  from google.appengine.tools.devappserver2 import blob_upload
  from google.appengine.tools.devappserver2 import channel
  from google.appengine.tools.devappserver2 import constants
  
  from google.appengine.tools.devappserver2 import endpoints
  from google.appengine.tools.devappserver2 import errors
  from google.appengine.tools.devappserver2 import file_watcher
  from google.appengine.tools.devappserver2 import go_runtime
  from google.appengine.tools.devappserver2 import http_runtime_constants
  from google.appengine.tools.devappserver2 import instance
  from google.appengine.tools.devappserver2 import login
  from google.appengine.tools.devappserver2 import php_runtime
  from google.appengine.tools.devappserver2 import python_runtime
  from google.appengine.tools.devappserver2 import request_rewriter
  from google.appengine.tools.devappserver2 import runtime_config_pb2
  from google.appengine.tools.devappserver2 import start_response_utils
  from google.appengine.tools.devappserver2 import static_files_handler
  from google.appengine.tools.devappserver2 import thread_executor
  from google.appengine.tools.devappserver2 import url_handler
  from google.appengine.tools.devappserver2 import util
  from google.appengine.tools.devappserver2 import wsgi_handler
  from google.appengine.tools.devappserver2 import wsgi_server
  
  
  _LOWER_HEX_DIGITS = string.hexdigits.lower()
  _UPPER_HEX_DIGITS = string.hexdigits.upper()
  _REQUEST_ID_HASH_LENGTH = 8
  
  _THREAD_POOL = thread_executor.ThreadExecutor()
  _RESTART_INSTANCES_CONFIG_CHANGES = frozenset(
      [application_configuration.NORMALIZED_LIBRARIES_CHANGED,
       application_configuration.SKIP_FILES_CHANGED,
       application_configuration.NOBUILD_FILES_CHANGED,
       # The server must be restarted when the handlers change because files
       # appearing in static content handlers make them unavailable to the
       # runtime.
       application_configuration.HANDLERS_CHANGED,
       application_configuration.ENV_VARIABLES_CHANGED])
  
  _REQUEST_LOGGING_BLACKLIST_RE = re.compile(
      r'^/_ah/(?:channel/(?:dev|jsapi)|img|login|upload)')
  
  # Fake arguments for _handle_script_request for request types that don't use
  # user-specified handlers.
  _EMPTY_MATCH = re.match('', '')
  _DUMMY_URLMAP = appinfo.URLMap(script='/')
  _SHUTDOWN_TIMEOUT = 30
  
  
  def _static_files_regex_from_handlers(handlers):
    patterns = []
    for url_map in handlers:
      handler_type = url_map.GetHandlerType()
      if url_map.application_readable:
        continue
      if handler_type == appinfo.STATIC_FILES:
        patterns.append(r'(%s)' % url_map.upload)
      elif handler_type == appinfo.STATIC_DIR:
        patterns.append('(%s%s%s)' % (url_map.static_dir.rstrip(os.path.sep),
                                      re.escape(os.path.sep), r'.*'))
    return r'^%s$' % '|'.join(patterns)
  
  
  class InteractiveCommandError(errors.Error):
    pass
  
  
  class _ScriptHandler(url_handler.UserConfiguredURLHandler):
    """A URL handler that will cause the request to be dispatched to an instance.
  
    This handler is special in that it does not have a working handle() method
    since the Module's dispatch logic is used to select the appropriate Instance.
    """
  
    def __init__(self, url_map):
      """Initializer for _ScriptHandler.
  
      Args:
        url_map: An appinfo.URLMap instance containing the configuration for this
            handler.
      """
      try:
        url_pattern = re.compile('%s$' % url_map.url)
      except re.error, e:
        raise errors.InvalidAppConfigError(
            'invalid url %r in script handler: %s' % (url_map.url, e))
  
      super(_ScriptHandler, self).__init__(url_map, url_pattern)
      self.url_map = url_map
  
    def handle(self, match, environ, start_response):
      """This is a dummy method that should never be called."""
      raise NotImplementedError()
  
  
  class Module(object):
    """The abstract base for all instance pool implementations."""
  
    _RUNTIME_INSTANCE_FACTORIES = {
  
        'go': go_runtime.GoRuntimeInstanceFactory,
        'php': php_runtime.PHPRuntimeInstanceFactory,
        'python': python_runtime.PythonRuntimeInstanceFactory,
        'python27': python_runtime.PythonRuntimeInstanceFactory,
    }
  
    def _create_instance_factory(self,
                                 module_configuration):
      """Create an instance.InstanceFactory.
  
      Args:
        module_configuration: An application_configuration.ModuleConfiguration
            instance storing the configuration data for a module.
  
      Returns:
        A instance.InstanceFactory subclass that can be used to create instances
        with the provided configuration.
  
      Raises:
        RuntimeError: if the configuration specifies an unknown runtime.
      """
      # TODO: a bad runtime should be caught before we get here.
      if module_configuration.runtime not in self._RUNTIME_INSTANCE_FACTORIES:
        raise RuntimeError(
            'Unknown runtime %r; supported runtimes are %s.' %
            (module_configuration.runtime,
             ', '.join(
                 sorted(repr(k) for k in self._RUNTIME_INSTANCE_FACTORIES))))
      instance_factory = self._RUNTIME_INSTANCE_FACTORIES[
          module_configuration.runtime]
      return instance_factory(
          request_data=self._request_data,
          runtime_config_getter=self._get_runtime_config,
          module_configuration=module_configuration)
  
    def _create_url_handlers(self):
      """Constructs URLHandlers based on the module configuration.
  
      Returns:
        A list of url_handler.URLHandlers corresponding that can react as
        described in the given configuration.
      """
      handlers = []
      # Add special URL handlers (taking precedence over user-defined handlers)
      url_pattern = '/%s$' % login.LOGIN_URL_RELATIVE
      handlers.append(wsgi_handler.WSGIHandler(login.application,
                                               url_pattern))
      url_pattern = '/%s' % blob_upload.UPLOAD_URL_PATH
      # The blobstore upload handler forwards successful requests back to self
      handlers.append(
          wsgi_handler.WSGIHandler(blob_upload.Application(self), url_pattern))
  
      url_pattern = '/%s' % blob_image.BLOBIMAGE_URL_PATTERN
      handlers.append(
          wsgi_handler.WSGIHandler(blob_image.Application(), url_pattern))
  
      url_pattern = '/%s' % channel.CHANNEL_URL_PATTERN
      handlers.append(
          wsgi_handler.WSGIHandler(channel.application, url_pattern))
  
      url_pattern = '/%s' % endpoints.API_SERVING_PATTERN
      handlers.append(
          wsgi_handler.WSGIHandler(
              endpoints.EndpointsDispatcher(self._dispatcher), url_pattern))
  
      found_start_handler = False
      found_warmup_handler = False
      # Add user-defined URL handlers
      for url_map in self._module_configuration.handlers:
        handler_type = url_map.GetHandlerType()
        if handler_type == appinfo.HANDLER_SCRIPT:
          handlers.append(_ScriptHandler(url_map))
          if not found_start_handler and re.match('%s$' % url_map.url,
                                                  '/_ah/start'):
            found_start_handler = True
          if not found_warmup_handler and re.match('%s$' % url_map.url,
                                                   '/_ah/warmup'):
            found_warmup_handler = True
        elif handler_type == appinfo.STATIC_FILES:
          handlers.append(
              static_files_handler.StaticFilesHandler(
                  self._module_configuration.application_root,
                  url_map))
        elif handler_type == appinfo.STATIC_DIR:
          handlers.append(
              static_files_handler.StaticDirHandler(
                  self._module_configuration.application_root,
                  url_map))
        else:
          assert 0, 'unexpected handler %r for %r' % (handler_type, url_map)
      # Add a handler for /_ah/start if no script handler matches.
      if not found_start_handler:
        handlers.insert(0, _ScriptHandler(self._instance_factory.START_URL_MAP))
      # Add a handler for /_ah/warmup if no script handler matches and warmup is
      # enabled.
      if (not found_warmup_handler and
          'warmup' in (self._module_configuration.inbound_services or [])):
        handlers.insert(0, _ScriptHandler(self._instance_factory.WARMUP_URL_MAP))
      return handlers
  
    def _get_runtime_config(self):
      """Returns the configuration for the runtime.
  
      Returns:
        A runtime_config_pb2.Config instance representing the configuration to be
        passed to an instance. NOTE: This does *not* include the instance_id
        field, which must be populated elsewhere.
      """
      runtime_config = runtime_config_pb2.Config()
      runtime_config.app_id = self._module_configuration.application
      runtime_config.version_id = self._module_configuration.version_id
      runtime_config.threadsafe = self._module_configuration.threadsafe or False
      runtime_config.application_root = (
          self._module_configuration.application_root)
      if not self._allow_skipped_files:
        runtime_config.skip_files = str(self._module_configuration.skip_files)
        runtime_config.static_files = _static_files_regex_from_handlers(
            self._module_configuration.handlers)
  
      # AppScale: Pack both API ports into the same field.
      if (self._external_api_port is not None and
          self._module_configuration.runtime == 'python27'):
        port_bytes = struct.pack('HH', self._api_port, self._external_api_port)
        runtime_config.api_port = struct.unpack('I', port_bytes)[0]
      else:
        runtime_config.api_port = self._api_port
  
      runtime_config.stderr_log_level = self._runtime_stderr_loglevel
      runtime_config.datacenter = 'us1'
      runtime_config.auth_domain = self._auth_domain
  
      for library in self._module_configuration.normalized_libraries:
        runtime_config.libraries.add(name=library.name, version=library.version)
  
      for key, value in (self._module_configuration.env_variables or {}).items():
        runtime_config.environ.add(key=str(key), value=str(value))
  
      if self._cloud_sql_config:
        runtime_config.cloud_sql_config.CopyFrom(self._cloud_sql_config)
  
      if self._module_configuration.runtime == 'php':
        if self._php_executable_path:
          runtime_config.php_config.php_executable_path = (
              self._php_executable_path)
        runtime_config.php_config.enable_debugger = (
            self._enable_php_remote_debugging)
      if (self._python_config and
          self._module_configuration.runtime.startswith('python')):
        runtime_config.python_config.CopyFrom(self._python_config)
      return runtime_config
  
    def _maybe_restart_instances(self, config_changed, file_changed):
      """Restarts instances. May avoid some restarts depending on policy.
  
      One of config_changed or file_changed must be True.
  
      Args:
        config_changed: True if the configuration for the application has changed.
        file_changed: True if any file relevant to the application has changed.
      """
      if not config_changed and not file_changed:
        return
  
      logging.debug('Restarting instances.')
      policy = self._instance_factory.FILE_CHANGE_INSTANCE_RESTART_POLICY
      assert policy is not None, 'FILE_CHANGE_INSTANCE_RESTART_POLICY not set'
  
      with self._condition:
        instances_to_quit = set()
        for inst in self._instances:
          if (config_changed or
              (policy == instance.ALWAYS) or
              (policy == instance.AFTER_FIRST_REQUEST and inst.total_requests)):
            instances_to_quit.add(inst)
        self._instances -= instances_to_quit
  
      for inst in instances_to_quit:
        inst.quit(allow_async=True)
  
    def _handle_changes(self):
      """Handle file or configuration changes."""
      # Always check for config and file changes because checking also clears
      # pending changes.
      config_changes = self._module_configuration.check_for_updates()
      has_file_changes = self._watcher.has_changes()
  
      if application_configuration.HANDLERS_CHANGED in config_changes:
        handlers = self._create_url_handlers()
        with self._handler_lock:
          self._handlers = handlers
  
      if has_file_changes:
        self._instance_factory.files_changed()
  
      if config_changes & _RESTART_INSTANCES_CONFIG_CHANGES:
        self._instance_factory.configuration_changed(config_changes)
  
      self._maybe_restart_instances(
          config_changed=bool(config_changes & _RESTART_INSTANCES_CONFIG_CHANGES),
          file_changed=has_file_changes)
  
    def __init__(self,
                 module_configuration,
                 host,
                 balanced_port,
                 api_port,
                 auth_domain,
                 runtime_stderr_loglevel,
                 php_executable_path,
                 enable_php_remote_debugging,
                 python_config,
                 cloud_sql_config,
                 default_version_port,
                 port_registry,
                 request_data,
                 dispatcher,
                 max_instances,
                 use_mtime_file_watcher,
                 automatic_restarts,
                 allow_skipped_files,
                 external_api_port=None):
      """Initializer for Module.
  
      Args:
        module_configuration: An application_configuration.ModuleConfiguration
            instance storing the configuration data for a module.
        host: A string containing the host that any HTTP servers should bind to
            e.g. "localhost".
        balanced_port: An int specifying the port where the balanced module for
            the pool should listen.
        api_port: The port that APIModule listens for RPC requests on.
        auth_domain: A string containing the auth domain to set in the environment
            variables.
        runtime_stderr_loglevel: An int reprenting the minimum logging level at
            which runtime log messages should be written to stderr. See
            devappserver2.py for possible values.
        php_executable_path: A string containing the path to PHP execution e.g.
            "/usr/bin/php-cgi".
        enable_php_remote_debugging: A boolean indicating whether the PHP
            interpreter should be started with XDebug remote debugging enabled.
        python_config: A runtime_config_pb2.PythonConfig instance containing
            Python runtime-specific configuration. If None then defaults are
            used.
        cloud_sql_config: A runtime_config_pb2.CloudSQL instance containing the
            required configuration for local Google Cloud SQL development. If None
            then Cloud SQL will not be available.
        default_version_port: An int containing the port of the default version.
        port_registry: A dispatcher.PortRegistry used to provide the Dispatcher
            with a mapping of port to Module and Instance.
        request_data: A wsgi_request_info.WSGIRequestInfo that will be provided
            with request information for use by API stubs.
        dispatcher: A Dispatcher instance that can be used to make HTTP requests.
        max_instances: The maximum number of instances to create for this module.
            If None then there is no limit on the number of created instances.
        use_mtime_file_watcher: A bool containing whether to use mtime polling to
            monitor file changes even if other options are available on the
            current platform.
        automatic_restarts: If True then instances will be restarted when a
            file or configuration change that effects them is detected.
        allow_skipped_files: If True then all files in the application's directory
            are readable, even if they appear in a static handler or "skip_files"
            directive.
        external_api_port: An integer specifying the location of an external API
            server.
      """
      self._module_configuration = module_configuration
      self._name = module_configuration.module_name
      self._host = host
      self._api_port = api_port
      self._external_api_port = external_api_port
      self._auth_domain = auth_domain
      self._runtime_stderr_loglevel = runtime_stderr_loglevel
      self._balanced_port = balanced_port
      self._php_executable_path = php_executable_path
      self._enable_php_remote_debugging = enable_php_remote_debugging
      self._python_config = python_config
      self._cloud_sql_config = cloud_sql_config
      self._request_data = request_data
      # _create_instance_factory() transitively calls _get_runtime_config, which
      # uses self._allow_skipped_files.
      self._allow_skipped_files = allow_skipped_files
      self._instance_factory = self._create_instance_factory(
          self._module_configuration)
      self._dispatcher = dispatcher
      self._max_instances = max_instances
      self._automatic_restarts = automatic_restarts
      self._use_mtime_file_watcher = use_mtime_file_watcher
      if self._automatic_restarts:
        self._watcher = file_watcher.get_file_watcher(
            [self._module_configuration.application_root] +
            self._instance_factory.get_restart_directories(),
            self._use_mtime_file_watcher)
      else:
        self._watcher = None
      self._handler_lock = threading.Lock()
      self._handlers = self._create_url_handlers()
      self._default_version_port = default_version_port
      self._port_registry = port_registry
  
      self._balanced_module = wsgi_server.WsgiServer(
          (self._host, self._balanced_port), self)
      self._quit_event = threading.Event()  # Set when quit() has been called.
  
      # AppScale: Keeps track of active requests in case of shutdown.
  
      # Indicates that the instance should refuse future requests.
      self.sigterm_sent = False
  
      # Handles request count and sigterm flag mutations.
      self.graceful_shutdown_lock = threading.Lock()
  
      # Keeps track of how many requests the instance is serving.
      self.request_count = 0
  
      # End AppScale.
  
    @property
    def name(self):
      """The name of the module, as defined in app.yaml.
  
      This value will be constant for the lifetime of the module even in the
      module configuration changes.
      """
      return self._name
  
    @property
    def ready(self):
      """The module is ready to handle HTTP requests."""
      return self._balanced_module.ready
  
    @property
    def balanced_port(self):
      """The port that the balanced HTTP server for the Module is listening on."""
      assert self._balanced_module.ready, 'balanced module not running'
      return self._balanced_module.port
  
    @property
    def host(self):
      """The host that the HTTP server(s) for this Module is listening on."""
      return self._host
  
    @property
    def balanced_address(self):
      """The address of the balanced HTTP server e.g. "localhost:8080"."""
      if self.balanced_port != 80:
        return '%s:%s' % (self.host, self.balanced_port)
      else:
        return self.host
  
    @property
    def max_instance_concurrent_requests(self):
      """The number of concurrent requests that each Instance can handle."""
      return self._instance_factory.max_concurrent_requests
  
    @property
    def module_configuration(self):
      """The application_configuration.ModuleConfiguration for this module."""
      return self._module_configuration
  
    @property
    def supports_interactive_commands(self):
      """True if the module can evaluate arbitrary code and return the result."""
      return self._instance_factory.SUPPORTS_INTERACTIVE_REQUESTS
  
    def _handle_script_request(self,
                               environ,
                               start_response,
                               url_map,
                               match,
                               inst=None):
      """Handles a HTTP request that has matched a script handler.
  
      Args:
        environ: An environ dict for the request as defined in PEP-333.
        start_response: A function with semantics defined in PEP-333.
        url_map: An appinfo.URLMap instance containing the configuration for the
            handler that matched.
        match: A re.MatchObject containing the result of the matched URL pattern.
        inst: The Instance to send the request to. If None then an appropriate
            Instance will be chosen.
  
      Returns:
        An iterable over strings containing the body of the HTTP response.
      """
      raise NotImplementedError()
  
    def _no_handler_for_request(self, environ, start_response, request_id):
      """Handle a HTTP request that does not match any user-defined handlers."""
      self._insert_log_message('No handlers matched this URL.', 2, request_id)
      start_response('404 Not Found', [('Content-Type', 'text/plain')])
      return ['The url "%s" does not match any handlers.' % environ['PATH_INFO']]
  
    def _error_response(self, environ, start_response, status):
      start_response('%d %s' % (status, httplib.responses[status]), [])
      return []
  
    # AppScale: Check if the instance should be shutting down before handling
    # request.
    def _handle_request(self, environ, start_response, **kwargs):
      """ A _handle_request wrapper that keeps track of active requests.
  
      Args:
        environ: An environ dict for the request as defined in PEP-333.
        start_response: A function with semantics defined in PEP-333.
      Returns:
        An iterable over strings containing the body of the HTTP response.
      """
      with self.graceful_shutdown_lock:
        if self.sigterm_sent:
          start_response('503 Service Unavailable',
                         [('Content-Type', 'text/plain')])
          return ['This instance is shutting down']
  
        self.request_count += 1
  
      try:
        return self._handle_request_impl(environ, start_response, **kwargs)
      finally:
        with self.graceful_shutdown_lock:
          self.request_count -= 1
  
    # End AppScale.
  
    def _handle_request_impl(self, environ, start_response, inst=None,
                             request_type=instance.NORMAL_REQUEST):
      """Handles a HTTP request.
  
      Args:
        environ: An environ dict for the request as defined in PEP-333.
        start_response: A function with semantics defined in PEP-333.
        inst: The Instance to send the request to. If None then an appropriate
            Instance will be chosen. Setting inst is not meaningful if the
            request does not match a "script" handler.
        request_type: The type of the request. See instance.*_REQUEST module
            constants.
  
      Returns:
        An iterable over strings containing the body of the HTTP response.
      """
      try:
        environ['SERVER_PORT'] = environ['HTTP_HOST'].split(':')[1]
      except IndexError:
        scheme = environ['HTTP_X_FORWARDED_PROTO']
        if scheme == 'http':
          environ['SERVER_PORT'] = 80
        else:
          environ['SERVER_PORT'] = 443
  
      if 'HTTP_HOST' in environ:
        environ['SERVER_NAME'] = environ['HTTP_HOST'].split(':', 1)[0]
      environ['DEFAULT_VERSION_HOSTNAME'] = '%s:%s' % (
        environ['SERVER_NAME'], environ['SERVER_PORT'])
      with self._request_data.request(
          environ,
          self._module_configuration) as request_id:
        should_log_request = not _REQUEST_LOGGING_BLACKLIST_RE.match(
            environ['PATH_INFO'])
        environ['REQUEST_ID_HASH'] = self.generate_request_id_hash()
        if should_log_request:
          environ['REQUEST_LOG_ID'] = self.generate_request_log_id()
          if 'HTTP_HOST' in environ:
            hostname = environ['HTTP_HOST']
          elif environ['SERVER_PORT'] == '80':
            hostname = environ['SERVER_NAME']
          else:
            hostname = '%s:%s' % (environ['SERVER_NAME'], environ['SERVER_PORT'])
  
          if environ.get('QUERY_STRING'):
            resource = '%s?%s' % (urllib.quote(environ['PATH_INFO']),
                                  environ['QUERY_STRING'])
          else:
            resource = urllib.quote(environ['PATH_INFO'])
          email, _, _ = login.get_user_info(environ.get('HTTP_COOKIE', ''))
          method = environ.get('REQUEST_METHOD', 'GET')
          http_version = environ.get('SERVER_PROTOCOL', 'HTTP/1.0')
  
          logservice = apiproxy_stub_map.apiproxy.GetStub('logservice')
          logservice.start_request(
              request_id=request_id,
              user_request_id=environ['REQUEST_LOG_ID'],
              ip=environ.get('REMOTE_ADDR', ''),
              app_id=self._module_configuration.application,
              version_id=self._module_configuration.version_id,
              nickname=email.split('@', 1)[0],
              user_agent=environ.get('HTTP_USER_AGENT', ''),
              host=hostname,
              method=method,
              resource=resource,
              http_version=http_version)
  
        def wrapped_start_response(status, response_headers, exc_info=None):
          response_headers.append(('Server',
                                   http_runtime_constants.SERVER_SOFTWARE))
          if should_log_request:
            headers = wsgiref.headers.Headers(response_headers)
            status_code = int(status.split(' ', 1)[0])
            content_length = int(headers.get('Content-Length', 0))
            logservice.end_request(request_id, status_code, content_length)
            logging.info('%(module_name)s: '
                         '"%(method)s %(resource)s %(http_version)s" '
                         '%(status)d %(content_length)s',
                         {'module_name': self.name,
                          'method': method,
                          'resource': resource,
                          'http_version': http_version,
                          'status': status_code,
                          'content_length': content_length or '-'})
          return start_response(status, response_headers, exc_info)
  
        if (environ['REQUEST_METHOD'] in ('GET', 'HEAD', 'TRACE') and
            int(environ.get('CONTENT_LENGTH') or '0') != 0):
          # CONTENT_LENGTH may be empty or absent.
          wrapped_start_response('400 Bad Request', [])
          return ['"%s" requests may not contain bodies.' %
                  environ['REQUEST_METHOD']]
  
        with self._handler_lock:
          handlers = self._handlers
  
        try:
          request_url = environ['PATH_INFO']
          if request_type in (instance.BACKGROUND_REQUEST,
                              instance.INTERACTIVE_REQUEST,
                              instance.SHUTDOWN_REQUEST):
            app = functools.partial(self._handle_script_request,
                                    url_map=_DUMMY_URLMAP,
                                    match=_EMPTY_MATCH,
                                    request_id=request_id,
                                    inst=inst,
                                    request_type=request_type)
            return request_rewriter.frontend_rewriter_middleware(app)(
                environ, wrapped_start_response)
          for handler in handlers:
            match = handler.match(request_url)
            if match:
              auth_failure = handler.handle_authorization(environ,
                                                          wrapped_start_response)
              if auth_failure is not None:
                return auth_failure
  
              if isinstance(handler, _ScriptHandler):
                app = functools.partial(self._handle_script_request,
                                        url_map=handler.url_map,
                                        match=match,
                                        request_id=request_id,
                                        inst=inst,
                                        request_type=request_type)
                return request_rewriter.frontend_rewriter_middleware(app)(
                    environ, wrapped_start_response)
              else:
                return handler.handle(match, environ, wrapped_start_response)
          return self._no_handler_for_request(environ, wrapped_start_response,
                                              request_id)
        except StandardError, e:
          logging.exception('Request to %r failed', request_url)
          wrapped_start_response('500 Internal Server Error', [], e)
          return []
  
    def _async_shutdown_instance(self, inst, port):
      _THREAD_POOL.submit(self._shutdown_instance, inst, port)
  
    def _shutdown_instance(self, inst, port):
      force_shutdown_time = time.time() + _SHUTDOWN_TIMEOUT
      try:
        environ = self.build_request_environ(
            'GET', '/_ah/stop', [], '', '0.1.0.3', port, fake_login=True)
        self._handle_request(environ,
                             start_response_utils.null_start_response,
                             inst=inst,
                             request_type=instance.SHUTDOWN_REQUEST)
        logging.debug('Sent shutdown request: %s', inst)
      except:
        logging.exception('Internal error while handling shutdown request.')
      finally:
        time_to_wait = force_shutdown_time - time.time()
        self._quit_event.wait(time_to_wait)
        inst.quit(force=True)
  
    def _insert_log_message(self, message, level, request_id):
      logs_group = log_service_pb.UserAppLogGroup()
      log_line = logs_group.add_log_line()
      log_line.set_timestamp_usec(int(time.time() * 1e6))
      log_line.set_level(level)
      log_line.set_message(message)
      request = log_service_pb.FlushRequest()
      request.set_logs(logs_group.Encode())
      response = api_base_pb.VoidProto()
      logservice = apiproxy_stub_map.apiproxy.GetStub('logservice')
      logservice._Dynamic_Flush(request, response, request_id)
  
    @staticmethod
    def generate_request_log_id():
      """Generate a random REQUEST_LOG_ID.
  
      Returns:
        A string suitable for use as a REQUEST_LOG_ID. The returned string is
        variable length to emulate the the production values, which encapsulate
        the application id, version and some log state.
      """
      return ''.join(random.choice(_LOWER_HEX_DIGITS)
                     for _ in range(random.randrange(30, 100)))
  
    @staticmethod
    def generate_request_id_hash():
      """Generate a random REQUEST_ID_HASH."""
      return ''.join(random.choice(_UPPER_HEX_DIGITS)
                     for _ in range(_REQUEST_ID_HASH_LENGTH))
  
    def set_num_instances(self, instances):
      """Sets the number of instances for this module to run.
  
      Args:
        instances: An int containing the number of instances to run.
      """
      raise request_info.NotSupportedWithAutoScalingError()
  
    def get_num_instances(self):
      """Returns the number of instances for this module to run."""
      raise request_info.NotSupportedWithAutoScalingError()
  
    def suspend(self):
      """Stops the module from serving requests."""
      raise request_info.NotSupportedWithAutoScalingError()
  
    def resume(self):
      """Restarts the module."""
      raise request_info.NotSupportedWithAutoScalingError()
  
    def get_instance_address(self, instance_id):
      """Returns the address of the HTTP server for an instance."""
      return '%s:%s' % (self.host, self.get_instance_port(instance_id))
  
    def get_instance_port(self, instance_id):
      """Returns the port of the HTTP server for an instance."""
      raise request_info.NotSupportedWithAutoScalingError()
  
    def get_instance(self, instance_id):
      """Returns the instance with the provided instance ID."""
      raise request_info.NotSupportedWithAutoScalingError()
  
    @property
    def supports_individually_addressable_instances(self):
      return False
  
    def create_interactive_command_module(self):
      """Returns a InteractiveCommandModule that can be sent user commands."""
      if self._instance_factory.SUPPORTS_INTERACTIVE_REQUESTS:
        return InteractiveCommandModule(self._module_configuration,
                                        self._host,
                                        self._balanced_port,
                                        self._api_port,
                                        self._auth_domain,
                                        self._runtime_stderr_loglevel,
                                        self._php_executable_path,
                                        self._enable_php_remote_debugging,
                                        self._python_config,
                                        self._cloud_sql_config,
                                        self._default_version_port,
                                        self._port_registry,
                                        self._request_data,
                                        self._dispatcher,
                                        self._use_mtime_file_watcher,
                                        self._allow_skipped_files)
      else:
        raise NotImplementedError('runtime does not support interactive commands')
  
    def build_request_environ(self, method, relative_url, headers, body,
                              source_ip, port, fake_login=False):
      if isinstance(body, unicode):
        body = body.encode('ascii')
  
      url = urlparse.urlsplit(relative_url)
      if port != 80:
        host = '%s:%s' % (self.host, port)
      else:
        host = self.host
      environ = {constants.FAKE_IS_ADMIN_HEADER: '1',
                 'CONTENT_LENGTH': str(len(body)),
                 'PATH_INFO': url.path,
                 'QUERY_STRING': url.query,
                 'REQUEST_METHOD': method,
                 'REMOTE_ADDR': source_ip,
                 'SERVER_NAME': self.host,
                 'SERVER_PORT': str(port),
                 'SERVER_PROTOCOL': 'HTTP/1.1',
                 'wsgi.version': (1, 0),
                 'wsgi.url_scheme': 'http',
                 'wsgi.errors': cStringIO.StringIO(),
                 'wsgi.multithread': True,
                 'wsgi.multiprocess': True,
                 'wsgi.input': cStringIO.StringIO(body)}
      if fake_login:
        environ[constants.FAKE_LOGGED_IN_HEADER] = '1'
      util.put_headers_in_environ(headers, environ)
      environ['HTTP_HOST'] = host
      return environ
  
  
  class AutoScalingModule(Module):
    """A pool of instances that is autoscaled based on traffic."""
  
    # The minimum number of seconds to wait, after quitting an idle instance,
    # before quitting another idle instance.
    _MIN_SECONDS_BETWEEN_QUITS = 60
    # The time horizon to use when calculating the number of instances required
    # to serve the current level of traffic.
    _REQUIRED_INSTANCE_WINDOW_SECONDS = 60
  
    _DEFAULT_AUTOMATIC_SCALING = appinfo.AutomaticScaling(
        min_pending_latency='0.1s',
        max_pending_latency='0.5s',
        min_idle_instances=1,
        max_idle_instances=1000)
  
    @staticmethod
    def _parse_pending_latency(timing):
      """Parse a pending latency string into a float of the value in seconds.
  
      Args:
        timing: A str of the form 1.0s or 1000ms.
  
      Returns:
        A float representation of the value in seconds.
      """
      if timing.endswith('ms'):
        return float(timing[:-2]) / 1000
      else:
        return float(timing[:-1])
  
    @classmethod
    def _populate_default_automatic_scaling(cls, automatic_scaling):
      for attribute in automatic_scaling.ATTRIBUTES:
        if getattr(automatic_scaling, attribute) in ('automatic', None):
          setattr(automatic_scaling, attribute,
                  getattr(cls._DEFAULT_AUTOMATIC_SCALING, attribute))
  
    def _process_automatic_scaling(self, automatic_scaling):
      if automatic_scaling:
        self._populate_default_automatic_scaling(automatic_scaling)
      else:
        automatic_scaling = self._DEFAULT_AUTOMATIC_SCALING
      self._min_pending_latency = self._parse_pending_latency(
          automatic_scaling.min_pending_latency)
      self._max_pending_latency = self._parse_pending_latency(
          automatic_scaling.max_pending_latency)
      self._min_idle_instances = int(automatic_scaling.min_idle_instances)
      self._max_idle_instances = int(automatic_scaling.max_idle_instances)
  
    def __init__(self,
                 module_configuration,
                 host,
                 balanced_port,
                 api_port,
                 auth_domain,
                 runtime_stderr_loglevel,
                 php_executable_path,
                 enable_php_remote_debugging,
                 python_config,
                 cloud_sql_config,
                 default_version_port,
                 port_registry,
                 request_data,
                 dispatcher,
                 max_instances,
                 use_mtime_file_watcher,
                 automatic_restarts,
                 allow_skipped_files,
                 external_api_port=None):
      """Initializer for AutoScalingModule.
  
      Args:
        module_configuration: An application_configuration.ModuleConfiguration
            instance storing the configuration data for a module.
        host: A string containing the host that any HTTP servers should bind to
            e.g. "localhost".
        balanced_port: An int specifying the port where the balanced module for
            the pool should listen.
        api_port: The port that APIServer listens for RPC requests on.
        auth_domain: A string containing the auth domain to set in the environment
            variables.
        runtime_stderr_loglevel: An int reprenting the minimum logging level at
            which runtime log messages should be written to stderr. See
            devappserver2.py for possible values.
        php_executable_path: A string containing the path to PHP execution e.g.
            "/usr/bin/php-cgi".
        enable_php_remote_debugging: A boolean indicating whether the PHP
            interpreter should be started with XDebug remote debugging enabled.
        python_config: A runtime_config_pb2.PythonConfig instance containing
            Python runtime-specific configuration. If None then defaults are
            used.
        cloud_sql_config: A runtime_config_pb2.CloudSQL instance containing the
            required configuration for local Google Cloud SQL development. If None
            then Cloud SQL will not be available.
        default_version_port: An int containing the port of the default version.
        port_registry: A dispatcher.PortRegistry used to provide the Dispatcher
            with a mapping of port to Module and Instance.
        request_data: A wsgi_request_info.WSGIRequestInfo that will be provided
            with request information for use by API stubs.
        dispatcher: A Dispatcher instance that can be used to make HTTP requests.
        max_instances: The maximum number of instances to create for this module.
            If None then there is no limit on the number of created instances.
        use_mtime_file_watcher: A bool containing whether to use mtime polling to
            monitor file changes even if other options are available on the
            current platform.
        automatic_restarts: If True then instances will be restarted when a
            file or configuration change that effects them is detected.
        allow_skipped_files: If True then all files in the application's directory
            are readable, even if they appear in a static handler or "skip_files"
            directive.
        external_api_port: An integer specifying the location of an external API
            server.
      """
      super(AutoScalingModule, self).__init__(module_configuration,
                                              host,
                                              balanced_port,
                                              api_port,
                                              auth_domain,
                                              runtime_stderr_loglevel,
                                              php_executable_path,
                                              enable_php_remote_debugging,
                                              python_config,
                                              cloud_sql_config,
                                              default_version_port,
                                              port_registry,
                                              request_data,
                                              dispatcher,
                                              max_instances,
                                              use_mtime_file_watcher,
                                              automatic_restarts,
                                              allow_skipped_files,
                                              external_api_port)
  
      self._process_automatic_scaling(
          self._module_configuration.automatic_scaling)
  
      self._instances = set()  # Protected by self._condition.
      # A deque containg (time, num_outstanding_instance_requests) 2-tuples.
      # This is used to track the maximum number of outstanding requests in a time
      # period. Protected by self._condition.
      self._outstanding_request_history = collections.deque()
      self._num_outstanding_instance_requests = 0  # Protected by self._condition.
      # The time when the last instance was quit in seconds since the epoch.
      self._last_instance_quit_time = 0  # Protected by self._condition.
  
      self._condition = threading.Condition()  # Protects instance state.
      self._instance_adjustment_thread = threading.Thread(
          target=self._loop_adjusting_instances)
  
    def start(self):
      """Start background management of the Module."""
      self._balanced_module.start()
      self._port_registry.add(self.balanced_port, self, None)
      if self._watcher:
        self._watcher.start()
      self._instance_adjustment_thread.start()
  
    def quit(self):
      """Stops the Module."""
      self._quit_event.set()
      self._instance_adjustment_thread.join()
      # The instance adjustment thread depends on the balanced module and the
      # watcher so wait for it exit before quitting them.
      if self._watcher:
        self._watcher.quit()
      self._balanced_module.quit()
      with self._condition:
        instances = self._instances
        self._instances = set()
        self._condition.notify_all()
      for inst in instances:
        inst.quit(force=True)
  
    @property
    def instances(self):
      """A set of all the instances currently in the Module."""
      with self._condition:
        return set(self._instances)
  
    @property
    def num_outstanding_instance_requests(self):
      """The number of requests that instances are currently handling."""
      with self._condition:
        return self._num_outstanding_instance_requests
  
    def _handle_instance_request(self,
                                 environ,
                                 start_response,
                                 url_map,
                                 match,
                                 request_id,
                                 inst,
                                 request_type):
      """Handles a request routed a particular Instance.
  
      Args:
        environ: An environ dict for the request as defined in PEP-333.
        start_response: A function with semantics defined in PEP-333.
        url_map: An appinfo.URLMap instance containing the configuration for the
            handler that matched.
        match: A re.MatchObject containing the result of the matched URL pattern.
        request_id: A unique string id associated with the request.
        inst: The instance.Instance to send the request to.
        request_type: The type of the request. See instance.*_REQUEST module
            constants.
  
      Returns:
        An iterable over strings containing the body of the HTTP response.
      """
      if request_type != instance.READY_REQUEST:
        with self._condition:
          self._num_outstanding_instance_requests += 1
          self._outstanding_request_history.append(
              (time.time(), self.num_outstanding_instance_requests))
      try:
        logging.debug('Dispatching request to %s', inst)
        return inst.handle(environ, start_response, url_map, match, request_id,
                           request_type)
      finally:
        with self._condition:
          if request_type != instance.READY_REQUEST:
            self._num_outstanding_instance_requests -= 1
          self._condition.notify()
  
    def _handle_script_request(self,
                               environ,
                               start_response,
                               url_map,
                               match,
                               request_id,
                               inst=None,
                               request_type=instance.NORMAL_REQUEST):
      """Handles a HTTP request that has matched a script handler.
  
      Args:
        environ: An environ dict for the request as defined in PEP-333.
        start_response: A function with semantics defined in PEP-333.
        url_map: An appinfo.URLMap instance containing the configuration for the
            handler that matched.
        match: A re.MatchObject containing the result of the matched URL pattern.
        request_id: A unique string id associated with the request.
        inst: The instance.Instance to send the request to. If None then an
            appropriate instance.Instance will be chosen.
        request_type: The type of the request. See instance.*_REQUEST module
            constants.
  
      Returns:
        An iterable over strings containing the body of the HTTP response.
      """
      if inst is not None:
        return self._handle_instance_request(
            environ, start_response, url_map, match, request_id, inst,
            request_type)
  
      with self._condition:
        self._num_outstanding_instance_requests += 1
        self._outstanding_request_history.append(
            (time.time(), self.num_outstanding_instance_requests))
  
      try:
        start_time = time.time()
        timeout_time = start_time + self._min_pending_latency
        # Loop until an instance is available to handle the request.
        while True:
          if self._quit_event.is_set():
            return self._error_response(environ, start_response, 404)
          inst = self._choose_instance(timeout_time)
          if not inst:
            inst = self._add_instance(permit_warmup=False)
            if not inst:
              # No instance is available nor can a new one be created, so loop
              # waiting for one to be free.
              timeout_time = time.time() + 0.2
              continue
  
          try:
            logging.debug('Dispatching request to %s after %0.4fs pending',
                          inst, time.time() - start_time)
            return inst.handle(environ,
                               start_response,
                               url_map,
                               match,
                               request_id,
                               request_type)
          except instance.CannotAcceptRequests:
            continue
      finally:
        with self._condition:
          self._num_outstanding_instance_requests -= 1
          self._condition.notify()
  
    def _add_instance(self, permit_warmup):
      """Creates and adds a new instance.Instance to the Module.
  
      Args:
        permit_warmup: If True then the new instance.Instance will be sent a new
            warmup request if it is configured to receive them.
  
      Returns:
        The newly created instance.Instance. Returns None if no new instance
        could be created because the maximum number of instances have already
        been created.
      """
      if self._max_instances is not None:
        with self._condition:
          if len(self._instances) >= self._max_instances:
            return None
  
      perform_warmup = permit_warmup and (
          'warmup' in (self._module_configuration.inbound_services or []))
  
      inst = self._instance_factory.new_instance(
          self.generate_instance_id(),
          expect_ready_request=perform_warmup)
  
      with self._condition:
        if self._quit_event.is_set():
          return None
        self._instances.add(inst)
  
      if not inst.start():
        return None
  
      if perform_warmup:
        self._async_warmup(inst)
      else:
        with self._condition:
          self._condition.notify(self.max_instance_concurrent_requests)
      logging.debug('Created instance: %s', inst)
      return inst
  
    @staticmethod
    def generate_instance_id():
      return ''.join(random.choice(_LOWER_HEX_DIGITS) for _ in range(36))
  
    def _warmup(self, inst):
      """Send a warmup request to the given instance."""
  
      try:
        environ = self.build_request_environ(
            'GET', '/_ah/warmup', [], '', '0.1.0.3', self.balanced_port,
            fake_login=True)
        self._handle_request(environ,
                             start_response_utils.null_start_response,
                             inst=inst,
                             request_type=instance.READY_REQUEST)
        with self._condition:
          self._condition.notify(self.max_instance_concurrent_requests)
      except:
        logging.exception('Internal error while handling warmup request.')
  
    def _async_warmup(self, inst):
      """Asynchronously send a markup request to the given Instance."""
      _THREAD_POOL.submit(self._warmup, inst)
  
    def _trim_outstanding_request_history(self):
      """Removes obsolete entries from _outstanding_request_history."""
      window_start = time.time() - self._REQUIRED_INSTANCE_WINDOW_SECONDS
      with self._condition:
        while self._outstanding_request_history:
          t, _ = self._outstanding_request_history[0]
          if t < window_start:
            self._outstanding_request_history.popleft()
          else:
            break
  
    def _get_num_required_instances(self):
      """Returns the number of Instances required to handle the request load."""
      with self._condition:
        self._trim_outstanding_request_history()
        if not self._outstanding_request_history:
          return 0
        else:
          peak_concurrent_requests = max(
              current_requests
              for (t, current_requests)
              in self._outstanding_request_history)
          return int(math.ceil(peak_concurrent_requests /
                               self.max_instance_concurrent_requests))
  
    def _split_instances(self):
      """Returns a 2-tuple representing the required and extra Instances.
  
      Returns:
        A 2-tuple of (required_instances, not_required_instances):
          required_instances: The set of the instance.Instances, in a state that
                              can handle requests, required to handle the current
                              request load.
          not_required_instances: The set of the Instances contained in this
                                  Module that not are not required.
      """
      with self._condition:
        num_required_instances = self._get_num_required_instances()
  
        available = [inst for inst in self._instances
                     if inst.can_accept_requests]
        available.sort(key=lambda inst: -inst.num_outstanding_requests)
  
        required = set(available[:num_required_instances])
        return required, self._instances - required
  
    def _choose_instance(self, timeout_time):
      """Returns the best Instance to handle a request or None if all are busy."""
      with self._condition:
        while time.time() < timeout_time:
          required_instances, not_required_instances = self._split_instances()
          if required_instances:
            # Pick the instance with the most remaining capacity to handle
            # requests.
            required_instances = sorted(
                required_instances,
                key=lambda inst: inst.remaining_request_capacity)
            if required_instances[-1].remaining_request_capacity:
              return required_instances[-1]
  
          available_instances = [inst for inst in not_required_instances
                                 if inst.remaining_request_capacity > 0 and
                                 inst.can_accept_requests]
          if available_instances:
            # Pick the instance with the *least* capacity to handle requests
            # to avoid using unnecessary idle instances.
            available_instances.sort(
                key=lambda instance: instance.num_outstanding_requests)
            return available_instances[-1]
          else:
            self._condition.wait(timeout_time - time.time())
      return None
  
    def _adjust_instances(self):
      """Creates new Instances or deletes idle Instances based on current load."""
      now = time.time()
      with self._condition:
        _, not_required_instances = self._split_instances()
  
      if len(not_required_instances) < self._min_idle_instances:
        self._add_instance(permit_warmup=True)
      elif (len(not_required_instances) > self._max_idle_instances and
            now >
            (self._last_instance_quit_time + self._MIN_SECONDS_BETWEEN_QUITS)):
        for inst in not_required_instances:
          if not inst.num_outstanding_requests:
            try:
              inst.quit()
            except instance.CannotQuitServingInstance:
              pass
            else:
              self._last_instance_quit_time = now
              logging.debug('Quit instance: %s', inst)
              with self._condition:
                self._instances.discard(inst)
                break
  
    def _loop_adjusting_instances(self):
      """Loops until the Module exits, reloading, adding or removing Instances."""
      while not self._quit_event.is_set():
        if self.ready:
          if self._automatic_restarts:
            self._handle_changes()
          self._adjust_instances()
        self._quit_event.wait(timeout=1)
  
    def __call__(self, environ, start_response):
      return self._handle_request(environ, start_response)
  
  
  class ManualScalingModule(Module):
    """A pool of instances that is manually-scaled."""
  
    _DEFAULT_MANUAL_SCALING = appinfo.ManualScaling(instances='1')
    _MAX_REQUEST_WAIT_TIME = 10
  
    @classmethod
    def _populate_default_manual_scaling(cls, manual_scaling):
      for attribute in manual_scaling.ATTRIBUTES:
        if getattr(manual_scaling, attribute) in ('manual', None):
          setattr(manual_scaling, attribute,
                  getattr(cls._DEFAULT_MANUAL_SCALING, attribute))
  
    def _process_manual_scaling(self, manual_scaling):
      if manual_scaling:
        self._populate_default_manual_scaling(manual_scaling)
      else:
        manual_scaling = self._DEFAULT_MANUAL_SCALING
      self._initial_num_instances = int(manual_scaling.instances)
  
    def __init__(self,
                 module_configuration,
                 host,
                 balanced_port,
                 api_port,
                 auth_domain,
                 runtime_stderr_loglevel,
                 php_executable_path,
                 enable_php_remote_debugging,
                 python_config,
                 cloud_sql_config,
                 default_version_port,
                 port_registry,
                 request_data,
                 dispatcher,
                 max_instances,
                 use_mtime_file_watcher,
                 automatic_restarts,
                 allow_skipped_files,
                 external_api_port=None):
      """Initializer for ManualScalingModule.
  
      Args:
        module_configuration: An application_configuration.ModuleConfiguration
            instance storing the configuration data for a module.
        host: A string containing the host that any HTTP servers should bind to
            e.g. "localhost".
        balanced_port: An int specifying the port where the balanced module for
            the pool should listen.
        api_port: The port that APIServer listens for RPC requests on.
        auth_domain: A string containing the auth domain to set in the environment
            variables.
        runtime_stderr_loglevel: An int reprenting the minimum logging level at
            which runtime log messages should be written to stderr. See
            devappserver2.py for possible values.
        php_executable_path: A string containing the path to PHP execution e.g.
            "/usr/bin/php-cgi".
        enable_php_remote_debugging: A boolean indicating whether the PHP
            interpreter should be started with XDebug remote debugging enabled.
        python_config: A runtime_config_pb2.PythonConfig instance containing
            Python runtime-specific configuration. If None then defaults are
            used.
        cloud_sql_config: A runtime_config_pb2.CloudSQL instance containing the
            required configuration for local Google Cloud SQL development. If None
            then Cloud SQL will not be available.
        default_version_port: An int containing the port of the default version.
        port_registry: A dispatcher.PortRegistry used to provide the Dispatcher
            with a mapping of port to Module and Instance.
        request_data: A wsgi_request_info.WSGIRequestInfo that will be provided
            with request information for use by API stubs.
        dispatcher: A Dispatcher instance that can be used to make HTTP requests.
        max_instances: The maximum number of instances to create for this module.
            If None then there is no limit on the number of created instances.
        use_mtime_file_watcher: A bool containing whether to use mtime polling to
            monitor file changes even if other options are available on the
            current platform.
        automatic_restarts: If True then instances will be restarted when a
            file or configuration change that effects them is detected.
        allow_skipped_files: If True then all files in the application's directory
            are readable, even if they appear in a static handler or "skip_files"
            directive.
        external_api_port: An integer specifying the location of an external API
            server.
      """
      super(ManualScalingModule, self).__init__(module_configuration,
                                                host,
                                                balanced_port,
                                                api_port,
                                                auth_domain,
                                                runtime_stderr_loglevel,
                                                php_executable_path,
                                                enable_php_remote_debugging,
                                                python_config,
                                                cloud_sql_config,
                                                default_version_port,
                                                port_registry,
                                                request_data,
                                                dispatcher,
                                                max_instances,
                                                use_mtime_file_watcher,
                                                automatic_restarts,
                                                allow_skipped_files,
                                                external_api_port)
  
      self._process_manual_scaling(module_configuration.manual_scaling)
  
      self._instances = []  # Protected by self._condition.
      self._wsgi_servers = []  # Protected by self._condition.
      # Whether the module has been stopped. Protected by self._condition.
      self._suspended = False
  
      self._condition = threading.Condition()  # Protects instance state.
  
      # Serializes operations that modify the serving state of or number of
      # instances.
      self._instances_change_lock = threading.RLock()
  
      self._change_watcher_thread = threading.Thread(
          target=self._loop_watching_for_changes)
  
    def start(self):
      """Start background management of the Module."""
      self._balanced_module.start()
      self._port_registry.add(self.balanced_port, self, None)
      if self._watcher:
        self._watcher.start()
      self._change_watcher_thread.start()
      with self._instances_change_lock:
        if self._max_instances is not None:
          initial_num_instances = min(self._max_instances,
                                      self._initial_num_instances)
        else:
          initial_num_instances = self._initial_num_instances
        for _ in xrange(initial_num_instances):
          self._add_instance()
  
    def quit(self):
      """Stops the Module."""
      self._quit_event.set()
      self._change_watcher_thread.join()
      # The instance adjustment thread depends on the balanced module and the
      # watcher so wait for it exit before quitting them.
      if self._watcher:
        self._watcher.quit()
      self._balanced_module.quit()
      for wsgi_servr in self._wsgi_servers:
        wsgi_servr.quit()
      with self._condition:
        instances = self._instances
        self._instances = []
        self._condition.notify_all()
      for inst in instances:
        inst.quit(force=True)
  
    def get_instance_port(self, instance_id):
      """Returns the port of the HTTP server for an instance."""
      try:
        instance_id = int(instance_id)
      except ValueError:
        raise request_info.InvalidInstanceIdError()
      with self._condition:
        if 0 <= instance_id < len(self._instances):
          wsgi_servr = self._wsgi_servers[instance_id]
        else:
          raise request_info.InvalidInstanceIdError()
      return wsgi_servr.port
  
    @property
    def instances(self):
      """A set of all the instances currently in the Module."""
      with self._condition:
        return set(self._instances)
  
    def _handle_instance_request(self,
                                 environ,
                                 start_response,
                                 url_map,
                                 match,
                                 request_id,
                                 inst,
                                 request_type):
      """Handles a request routed a particular Instance.
  
      Args:
        environ: An environ dict for the request as defined in PEP-333.
        start_response: A function with semantics defined in PEP-333.
        url_map: An appinfo.URLMap instance containing the configuration for the
            handler that matched.
        match: A re.MatchObject containing the result of the matched URL pattern.
        request_id: A unique string id associated with the request.
        inst: The instance.Instance to send the request to.
        request_type: The type of the request. See instance.*_REQUEST module
            constants.
  
      Returns:
        An iterable over strings containing the body of the HTTP response.
      """
      start_time = time.time()
      timeout_time = start_time + self._MAX_REQUEST_WAIT_TIME
      try:
        while time.time() < timeout_time:
          logging.debug('Dispatching request to %s after %0.4fs pending',
                        inst, time.time() - start_time)
          try:
            return inst.handle(environ, start_response, url_map, match,
                               request_id, request_type)
          except instance.CannotAcceptRequests:
            pass
          inst.wait(timeout_time)
          if inst.has_quit:
            return self._error_response(environ, start_response, 503)
        else:
          return self._error_response(environ, start_response, 503)
      finally:
        with self._condition:
          self._condition.notify()
  
    def _handle_script_request(self,
                               environ,
                               start_response,
                               url_map,
                               match,
                               request_id,
                               inst=None,
                               request_type=instance.NORMAL_REQUEST):
      """Handles a HTTP request that has matched a script handler.
  
      Args:
        environ: An environ dict for the request as defined in PEP-333.
        start_response: A function with semantics defined in PEP-333.
        url_map: An appinfo.URLMap instance containing the configuration for the
            handler that matched.
        match: A re.MatchObject containing the result of the matched URL pattern.
        request_id: A unique string id associated with the request.
        inst: The instance.Instance to send the request to. If None then an
            appropriate instance.Instance will be chosen.
        request_type: The type of the request. See instance.*_REQUEST module
            constants.
  
      Returns:
        An iterable over strings containing the body of the HTTP response.
      """
      if ((request_type in (instance.NORMAL_REQUEST, instance.READY_REQUEST) and
           self._suspended) or self._quit_event.is_set()):
        return self._error_response(environ, start_response, 404)
      if self._module_configuration.is_backend:
        environ['BACKEND_ID'] = self._module_configuration.module_name
      else:
        environ['BACKEND_ID'] = (
            self._module_configuration.version_id.split('.', 1)[0])
      if inst is not None:
        return self._handle_instance_request(
            environ, start_response, url_map, match, request_id, inst,
            request_type)
  
      start_time = time.time()
      timeout_time = start_time + self._MAX_REQUEST_WAIT_TIME
      while time.time() < timeout_time:
        if ((request_type in (instance.NORMAL_REQUEST, instance.READY_REQUEST) and
             self._suspended) or self._quit_event.is_set()):
          return self._error_response(environ, start_response, 404)
        inst = self._choose_instance(timeout_time)
        if inst:
          try:
            logging.debug('Dispatching request to %s after %0.4fs pending',
                          inst, time.time() - start_time)
            return inst.handle(environ, start_response, url_map, match,
                               request_id, request_type)
          except instance.CannotAcceptRequests:
            continue
          finally:
            with self._condition:
              self._condition.notify()
      else:
        return self._error_response(environ, start_response, 503)
  
    def _add_instance(self):
      """Creates and adds a new instance.Instance to the Module.
  
      This must be called with _instances_change_lock held.
      """
      instance_id = self.get_num_instances()
      assert self._max_instances is None or instance_id < self._max_instances
      inst = self._instance_factory.new_instance(instance_id,
                                                 expect_ready_request=True)
      wsgi_servr = wsgi_server.WsgiServer(
          (self._host, 0), functools.partial(self._handle_request, inst=inst))
      wsgi_servr.start()
      self._port_registry.add(wsgi_servr.port, self, inst)
      with self._condition:
        if self._quit_event.is_set():
          return
        self._wsgi_servers.append(wsgi_servr)
        self._instances.append(inst)
        suspended = self._suspended
      if not suspended:
        self._async_start_instance(wsgi_servr, inst)
  
    def _async_start_instance(self, wsgi_servr, inst):
      _THREAD_POOL.submit(self._start_instance, wsgi_servr, inst)
  
    def _start_instance(self, wsgi_servr, inst):
      if inst.start():
        logging.debug('Started instance: %s at http://%s:%s', inst, self.host,
                      wsgi_servr.port)
        try:
          environ = self.build_request_environ(
              'GET', '/_ah/start', [], '', '0.1.0.3', wsgi_servr.port,
              fake_login=True)
          self._handle_request(environ,
                               start_response_utils.null_start_response,
                               inst=inst,
                               request_type=instance.READY_REQUEST)
          logging.debug('Sent start request: %s', inst)
          with self._condition:
            self._condition.notify(self.max_instance_concurrent_requests)
        except:
          logging.exception('Internal error while handling start request.')
  
    def _choose_instance(self, timeout_time):
      """Returns an Instance to handle a request or None if all are busy."""
      with self._condition:
        while time.time() < timeout_time:
          for inst in self._instances:
            if inst.can_accept_requests:
              return inst
          self._condition.wait(timeout_time - time.time())
        return None
  
    def _handle_changes(self):
      """Handle file or configuration changes."""
      # Always check for config and file changes because checking also clears
      # pending changes.
      config_changes = self._module_configuration.check_for_updates()
      has_file_changes = self._watcher.has_changes()
  
      if application_configuration.HANDLERS_CHANGED in config_changes:
        handlers = self._create_url_handlers()
        with self._handler_lock:
          self._handlers = handlers
  
      if has_file_changes:
        self._instance_factory.files_changed()
  
      if config_changes & _RESTART_INSTANCES_CONFIG_CHANGES:
        self._instance_factory.configuration_changed(config_changes)
  
      if config_changes & _RESTART_INSTANCES_CONFIG_CHANGES or has_file_changes:
        with self._instances_change_lock:
          if not self._suspended:
            self.restart()
  
    def _loop_watching_for_changes(self):
      """Loops until the InstancePool is done watching for file changes."""
      while not self._quit_event.is_set():
        if self.ready:
          if self._automatic_restarts:
            self._handle_changes()
        self._quit_event.wait(timeout=1)
  
    def get_num_instances(self):
      with self._instances_change_lock:
        with self._condition:
          return len(self._instances)
  
    def set_num_instances(self, instances):
      if self._max_instances is not None:
        instances = min(instances, self._max_instances)
  
      with self._instances_change_lock:
        with self._condition:
          running_instances = self.get_num_instances()
          if running_instances > instances:
            wsgi_servers_to_quit = self._wsgi_servers[instances:]
            del self._wsgi_servers[instances:]
            instances_to_quit = self._instances[instances:]
            del self._instances[instances:]
        if running_instances < instances:
          for _ in xrange(instances - running_instances):
            self._add_instance()
      if running_instances > instances:
        for inst, wsgi_servr in zip(instances_to_quit, wsgi_servers_to_quit):
          self._async_quit_instance(inst, wsgi_servr)
  
    def _async_quit_instance(self, inst, wsgi_servr):
      _THREAD_POOL.submit(self._quit_instance, inst, wsgi_servr)
  
    def _quit_instance(self, inst, wsgi_servr):
      port = wsgi_servr.port
      wsgi_servr.quit()
      inst.quit(expect_shutdown=True)
      self._shutdown_instance(inst, port)
  
    def suspend(self):
      """Suspends serving for this module, quitting all running instances."""
      with self._instances_change_lock:
        if self._suspended:
          raise request_info.ModuleAlreadyStoppedError()
        self._suspended = True
        with self._condition:
          instances_to_stop = zip(self._instances, self._wsgi_servers)
          for wsgi_servr in self._wsgi_servers:
            wsgi_servr.set_error(404)
      for inst, wsgi_servr in instances_to_stop:
        self._async_suspend_instance(inst, wsgi_servr.port)
  
    def _async_suspend_instance(self, inst, port):
      _THREAD_POOL.submit(self._suspend_instance, inst, port)
  
    def _suspend_instance(self, inst, port):
      inst.quit(expect_shutdown=True)
      self._shutdown_instance(inst, port)
  
    def resume(self):
      """Resumes serving for this module."""
      with self._instances_change_lock:
        if not self._suspended:
          raise request_info.ModuleAlreadyStartedError()
        self._suspended = False
        with self._condition:
          if self._quit_event.is_set():
            return
          wsgi_servers = self._wsgi_servers
        instances_to_start = []
        for instance_id, wsgi_servr in enumerate(wsgi_servers):
          inst = self._instance_factory.new_instance(instance_id,
                                                     expect_ready_request=True)
          wsgi_servr.set_app(functools.partial(self._handle_request, inst=inst))
          self._port_registry.add(wsgi_servr.port, self, inst)
          with self._condition:
            if self._quit_event.is_set():
              return
            self._instances[instance_id] = inst
  
          instances_to_start.append((wsgi_servr, inst))
      for wsgi_servr, inst in instances_to_start:
        self._async_start_instance(wsgi_servr, inst)
  
    def restart(self):
      """Restarts the the module, replacing all running instances."""
      with self._instances_change_lock:
        with self._condition:
          if self._quit_event.is_set():
            return
          instances_to_stop = self._instances[:]
          wsgi_servers = self._wsgi_servers[:]
        instances_to_start = []
        for instance_id, wsgi_servr in enumerate(wsgi_servers):
          inst = self._instance_factory.new_instance(instance_id,
                                                     expect_ready_request=True)
          wsgi_servr.set_app(functools.partial(self._handle_request, inst=inst))
          self._port_registry.add(wsgi_servr.port, self, inst)
          instances_to_start.append(inst)
        with self._condition:
          if self._quit_event.is_set():
            return
          self._instances[:] = instances_to_start
      for inst, wsgi_servr in zip(instances_to_stop, wsgi_servers):
        self._async_suspend_instance(inst, wsgi_servr.port)
      for wsgi_servr, inst in zip(wsgi_servers, instances_to_start):
        self._async_start_instance(wsgi_servr, inst)
  
    def get_instance(self, instance_id):
      """Returns the instance with the provided instance ID."""
      try:
        with self._condition:
          return self._instances[int(instance_id)]
      except (ValueError, IndexError):
        raise request_info.InvalidInstanceIdError()
  
    def __call__(self, environ, start_response, inst=None):
      return self._handle_request(environ, start_response, inst)
  
    @property
    def supports_individually_addressable_instances(self):
      return True
  
  
  class BasicScalingModule(Module):
    """A pool of instances that is basic-scaled."""
  
    _DEFAULT_BASIC_SCALING = appinfo.BasicScaling(max_instances='1',
                                                  idle_timeout='15m')
    _MAX_REQUEST_WAIT_TIME = 10
  
    @staticmethod
    def _parse_idle_timeout(timing):
      """Parse a idle timeout string into an int of the value in seconds.
  
      Args:
        timing: A str of the form 1m or 10s.
  
      Returns:
        An int representation of the value in seconds.
      """
      if timing.endswith('m'):
        return int(timing[:-1]) * 60
      else:
        return int(timing[:-1])
  
    @classmethod
    def _populate_default_basic_scaling(cls, basic_scaling):
      for attribute in basic_scaling.ATTRIBUTES:
        if getattr(basic_scaling, attribute) in ('basic', None):
          setattr(basic_scaling, attribute,
                  getattr(cls._DEFAULT_BASIC_SCALING, attribute))
  
    def _process_basic_scaling(self, basic_scaling):
      if basic_scaling:
        self._populate_default_basic_scaling(basic_scaling)
      else:
        basic_scaling = self._DEFAULT_BASIC_SCALING
      if self._max_instances is not None:
        self._max_instances = min(self._max_instances,
                                  int(basic_scaling.max_instances))
      else:
        self._max_instances = int(basic_scaling.max_instances)
      self._instance_idle_timeout = self._parse_idle_timeout(
          basic_scaling.idle_timeout)
  
    def __init__(self,
                 module_configuration,
                 host,
                 balanced_port,
                 api_port,
                 auth_domain,
                 runtime_stderr_loglevel,
                 php_executable_path,
                 enable_php_remote_debugging,
                 python_config,
                 cloud_sql_config,
                 default_version_port,
                 port_registry,
                 request_data,
                 dispatcher,
                 max_instances,
                 use_mtime_file_watcher,
                 automatic_restarts,
                 allow_skipped_files,
                 external_api_port=None):
      """Initializer for BasicScalingModule.
  
      Args:
        module_configuration: An application_configuration.ModuleConfiguration
            instance storing the configuration data for a module.
        host: A string containing the host that any HTTP servers should bind to
            e.g. "localhost".
        balanced_port: An int specifying the port where the balanced module for
            the pool should listen.
        api_port: The port that APIServer listens for RPC requests on.
        auth_domain: A string containing the auth domain to set in the environment
            variables.
        runtime_stderr_loglevel: An int reprenting the minimum logging level at
            which runtime log messages should be written to stderr. See
            devappserver2.py for possible values.
        php_executable_path: A string containing the path to PHP execution e.g.
            "/usr/bin/php-cgi".
        enable_php_remote_debugging: A boolean indicating whether the PHP
            interpreter should be started with XDebug remote debugging enabled.
        python_config: A runtime_config_pb2.PythonConfig instance containing
            Python runtime-specific configuration. If None then defaults are
            used.
        cloud_sql_config: A runtime_config_pb2.CloudSQL instance containing the
            required configuration for local Google Cloud SQL development. If None
            then Cloud SQL will not be available.
        default_version_port: An int containing the port of the default version.
        port_registry: A dispatcher.PortRegistry used to provide the Dispatcher
            with a mapping of port to Module and Instance.
        request_data: A wsgi_request_info.WSGIRequestInfo that will be provided
            with request information for use by API stubs.
        dispatcher: A Dispatcher instance that can be used to make HTTP requests.
        max_instances: The maximum number of instances to create for this module.
            If None then there is no limit on the number of created instances.
        use_mtime_file_watcher: A bool containing whether to use mtime polling to
            monitor file changes even if other options are available on the
            current platform.
        automatic_restarts: If True then instances will be restarted when a
            file or configuration change that effects them is detected.
        allow_skipped_files: If True then all files in the application's directory
            are readable, even if they appear in a static handler or "skip_files"
            directive.
        external_api_port: An integer specifying the location of an external API
            server.
      """
      super(BasicScalingModule, self).__init__(module_configuration,
                                               host,
                                               balanced_port,
                                               api_port,
                                               auth_domain,
                                               runtime_stderr_loglevel,
                                               php_executable_path,
                                               enable_php_remote_debugging,
                                               python_config,
                                               cloud_sql_config,
                                               default_version_port,
                                               port_registry,
                                               request_data,
                                               dispatcher,
                                               max_instances,
                                               use_mtime_file_watcher,
                                               automatic_restarts,
                                               allow_skipped_files,
                                               external_api_port)
      self._process_basic_scaling(module_configuration.basic_scaling)
  
      self._instances = []  # Protected by self._condition.
      self._wsgi_servers = []  # Protected by self._condition.
      # A list of booleans signifying whether the corresponding instance in
      # self._instances has been or is being started.
      self._instance_running = []  # Protected by self._condition.
  
      for instance_id in xrange(self._max_instances):
        inst = self._instance_factory.new_instance(instance_id,
                                                   expect_ready_request=True)
        self._instances.append(inst)
        self._wsgi_servers.append(wsgi_server.WsgiServer(
            (self._host, 0), functools.partial(self._handle_request, inst=inst)))
        self._instance_running.append(False)
  
      self._condition = threading.Condition()  # Protects instance state.
  
      self._change_watcher_thread = threading.Thread(
          target=self._loop_watching_for_changes_and_idle_instances)
  
    def start(self):
      """Start background management of the Module."""
      self._balanced_module.start()
      self._port_registry.add(self.balanced_port, self, None)
      if self._watcher:
        self._watcher.start()
      self._change_watcher_thread.start()
      for wsgi_servr, inst in zip(self._wsgi_servers, self._instances):
        wsgi_servr.start()
        self._port_registry.add(wsgi_servr.port, self, inst)
  
    def quit(self):
      """Stops the Module."""
      self._quit_event.set()
      self._change_watcher_thread.join()
      # The instance adjustment thread depends on the balanced module and the
      # watcher so wait for it exit before quitting them.
      if self._watcher:
        self._watcher.quit()
      self._balanced_module.quit()
      for wsgi_servr in self._wsgi_servers:
        wsgi_servr.quit()
      with self._condition:
        instances = self._instances
        self._instances = []
        self._condition.notify_all()
      for inst in instances:
        inst.quit(force=True)
  
    def get_instance_port(self, instance_id):
      """Returns the port of the HTTP server for an instance."""
      try:
        instance_id = int(instance_id)
      except ValueError:
        raise request_info.InvalidInstanceIdError()
      with self._condition:
        if 0 <= instance_id < len(self._instances):
          wsgi_servr = self._wsgi_servers[instance_id]
        else:
          raise request_info.InvalidInstanceIdError()
      return wsgi_servr.port
  
    @property
    def instances(self):
      """A set of all the instances currently in the Module."""
      with self._condition:
        return set(self._instances)
  
    def _handle_instance_request(self,
                                 environ,
                                 start_response,
                                 url_map,
                                 match,
                                 request_id,
                                 inst,
                                 request_type):
      """Handles a request routed a particular Instance.
  
      Args:
        environ: An environ dict for the request as defined in PEP-333.
        start_response: A function with semantics defined in PEP-333.
        url_map: An appinfo.URLMap instance containing the configuration for the
            handler that matched.
        match: A re.MatchObject containing the result of the matched URL pattern.
        request_id: A unique string id associated with the request.
        inst: The instance.Instance to send the request to.
        request_type: The type of the request. See instance.*_REQUEST module
            constants.
  
      Returns:
        An iterable over strings containing the body of the HTTP response.
      """
      instance_id = inst.instance_id
      start_time = time.time()
      timeout_time = start_time + self._MAX_REQUEST_WAIT_TIME
      try:
        while time.time() < timeout_time:
          logging.debug('Dispatching request to %s after %0.4fs pending',
                        inst, time.time() - start_time)
          try:
            return inst.handle(environ, start_response, url_map, match,
                               request_id, request_type)
          except instance.CannotAcceptRequests:
            pass
          if inst.has_quit:
            return self._error_response(environ, start_response, 503)
          with self._condition:
            if self._instance_running[instance_id]:
              should_start = False
            else:
              self._instance_running[instance_id] = True
              should_start = True
          if should_start:
            self._start_instance(instance_id)
          else:
            inst.wait(timeout_time)
        else:
          return self._error_response(environ, start_response, 503)
      finally:
        with self._condition:
          self._condition.notify()
  
    def _handle_script_request(self,
                               environ,
                               start_response,
                               url_map,
                               match,
                               request_id,
                               inst=None,
                               request_type=instance.NORMAL_REQUEST):
      """Handles a HTTP request that has matched a script handler.
  
      Args:
        environ: An environ dict for the request as defined in PEP-333.
        start_response: A function with semantics defined in PEP-333.
        url_map: An appinfo.URLMap instance containing the configuration for the
            handler that matched.
        match: A re.MatchObject containing the result of the matched URL pattern.
        request_id: A unique string id associated with the request.
        inst: The instance.Instance to send the request to. If None then an
            appropriate instance.Instance will be chosen.
        request_type: The type of the request. See instance.*_REQUEST module
            constants.
  
      Returns:
        An iterable over strings containing the body of the HTTP response.
      """
      if self._quit_event.is_set():
        return self._error_response(environ, start_response, 404)
      if self._module_configuration.is_backend:
        environ['BACKEND_ID'] = self._module_configuration.module_name
      else:
        environ['BACKEND_ID'] = (
            self._module_configuration.version_id.split('.', 1)[0])
      if inst is not None:
        return self._handle_instance_request(
            environ, start_response, url_map, match, request_id, inst,
            request_type)
  
      start_time = time.time()
      timeout_time = start_time + self._MAX_REQUEST_WAIT_TIME
      while time.time() < timeout_time:
        if self._quit_event.is_set():
          return self._error_response(environ, start_response, 404)
        inst = self._choose_instance(timeout_time)
        if inst:
          try:
            logging.debug('Dispatching request to %s after %0.4fs pending',
                          inst, time.time() - start_time)
            return inst.handle(environ, start_response, url_map, match,
                               request_id, request_type)
          except instance.CannotAcceptRequests:
            continue
          finally:
            with self._condition:
              self._condition.notify()
      else:
        return self._error_response(environ, start_response, 503)
  
    def _start_any_instance(self):
      """Choose an inactive instance and start it asynchronously.
  
      Returns:
        An instance.Instance that will be started asynchronously or None if all
        instances are already running.
      """
      with self._condition:
        for instance_id, running in enumerate(self._instance_running):
          if not running:
            self._instance_running[instance_id] = True
            inst = self._instances[instance_id]
            break
        else:
          return None
      self._async_start_instance(instance_id)
      return inst
  
    def _async_start_instance(self, instance_id):
      _THREAD_POOL.submit(self._start_instance, instance_id)
  
    def _start_instance(self, instance_id):
      with self._condition:
        if self._quit_event.is_set():
          return
        wsgi_servr = self._wsgi_servers[instance_id]
        inst = self._instances[instance_id]
      if inst.start():
        logging.debug('Started instance: %s at http://%s:%s', inst, self.host,
                      wsgi_servr.port)
        try:
          environ = self.build_request_environ(
              'GET', '/_ah/start', [], '', '0.1.0.3', wsgi_servr.port,
              fake_login=True)
          self._handle_request(environ,
                               start_response_utils.null_start_response,
                               inst=inst,
                               request_type=instance.READY_REQUEST)
          logging.debug('Sent start request: %s', inst)
          with self._condition:
            self._condition.notify(self.max_instance_concurrent_requests)
        except:
          logging.exception('Internal error while handling start request.')
  
    def _choose_instance(self, timeout_time):
      """Returns an Instance to handle a request or None if all are busy."""
      with self._condition:
        while time.time() < timeout_time and not self._quit_event.is_set():
          for inst in self._instances:
            if inst.can_accept_requests:
              return inst
          else:
            inst = self._start_any_instance()
            if inst:
              break
            self._condition.wait(timeout_time - time.time())
        else:
          return None
      if inst:
        inst.wait(timeout_time)
      return inst
  
    def _handle_changes(self):
      """Handle file or configuration changes."""
      # Always check for config and file changes because checking also clears
      # pending changes.
      config_changes = self._module_configuration.check_for_updates()
      has_file_changes = self._watcher.has_changes()
  
      if application_configuration.HANDLERS_CHANGED in config_changes:
        handlers = self._create_url_handlers()
        with self._handler_lock:
          self._handlers = handlers
  
      if has_file_changes:
        self._instance_factory.files_changed()
  
      if config_changes & _RESTART_INSTANCES_CONFIG_CHANGES:
        self._instance_factory.configuration_changed(config_changes)
  
      if config_changes & _RESTART_INSTANCES_CONFIG_CHANGES or has_file_changes:
        self.restart()
  
    def _loop_watching_for_changes_and_idle_instances(self):
      """Loops until the InstancePool is done watching for file changes."""
      while not self._quit_event.is_set():
        if self.ready:
          self._shutdown_idle_instances()
          if self._automatic_restarts:
            self._handle_changes()
        self._quit_event.wait(timeout=1)
  
    def _shutdown_idle_instances(self):
      instances_to_stop = []
      with self._condition:
        for instance_id, inst in enumerate(self._instances):
          if (self._instance_running[instance_id] and
              inst.idle_seconds > self._instance_idle_timeout):
            instances_to_stop.append((self._instances[instance_id],
                                      self._wsgi_servers[instance_id]))
            self._instance_running[instance_id] = False
            new_instance = self._instance_factory.new_instance(
                instance_id, expect_ready_request=True)
            self._instances[instance_id] = new_instance
            wsgi_servr = self._wsgi_servers[instance_id]
            wsgi_servr.set_app(
                functools.partial(self._handle_request, inst=new_instance))
            self._port_registry.add(wsgi_servr.port, self, new_instance)
      for inst, wsgi_servr in instances_to_stop:
        logging.debug('Shutting down %r', inst)
        self._stop_instance(inst, wsgi_servr)
  
    def _stop_instance(self, inst, wsgi_servr):
      inst.quit(expect_shutdown=True)
      self._async_shutdown_instance(inst, wsgi_servr.port)
  
    def restart(self):
      """Restarts the the module, replacing all running instances."""
      instances_to_stop = []
      instances_to_start = []
      with self._condition:
        if self._quit_event.is_set():
          return
        for instance_id, inst in enumerate(self._instances):
          if self._instance_running[instance_id]:
            instances_to_stop.append((inst, self._wsgi_servers[instance_id]))
            new_instance = self._instance_factory.new_instance(
                instance_id, expect_ready_request=True)
            self._instances[instance_id] = new_instance
            instances_to_start.append(instance_id)
            wsgi_servr = self._wsgi_servers[instance_id]
            wsgi_servr.set_app(
                functools.partial(self._handle_request, inst=new_instance))
            self._port_registry.add(wsgi_servr.port, self, new_instance)
      for instance_id in instances_to_start:
        self._async_start_instance(instance_id)
      for inst, wsgi_servr in instances_to_stop:
        self._stop_instance(inst, wsgi_servr)
  
    def get_instance(self, instance_id):
      """Returns the instance with the provided instance ID."""
      try:
        with self._condition:
          return self._instances[int(instance_id)]
      except (ValueError, IndexError):
        raise request_info.InvalidInstanceIdError()
  
    def __call__(self, environ, start_response, inst=None):
      return self._handle_request(environ, start_response, inst)
  
    @property
    def supports_individually_addressable_instances(self):
      return True
  
  
  class InteractiveCommandModule(Module):
    """A Module that can evaluate user commands.
  
    This module manages a single Instance which is started lazily.
    """
  
    _MAX_REQUEST_WAIT_TIME = 15
  
    def __init__(self,
                 module_configuration,
                 host,
                 balanced_port,
                 api_port,
                 auth_domain,
                 runtime_stderr_loglevel,
                 php_executable_path,
                 enable_php_remote_debugging,
                 python_config,
                 cloud_sql_config,
                 default_version_port,
                 port_registry,
                 request_data,
                 dispatcher,
                 use_mtime_file_watcher,
                 allow_skipped_files):
      """Initializer for InteractiveCommandModule.
  
      Args:
        module_configuration: An application_configuration.ModuleConfiguration
            instance storing the configuration data for this module.
        host: A string containing the host that will be used when constructing
            HTTP headers sent to the Instance executing the interactive command
            e.g. "localhost".
        balanced_port: An int specifying the port that will be used when
            constructing HTTP headers sent to the Instance executing the
            interactive command e.g. "localhost".
        api_port: The port that APIServer listens for RPC requests on.
        auth_domain: A string containing the auth domain to set in the environment
            variables.
        runtime_stderr_loglevel: An int reprenting the minimum logging level at
            which runtime log messages should be written to stderr. See
            devappserver2.py for possible values.
        php_executable_path: A string containing the path to PHP execution e.g.
            "/usr/bin/php-cgi".
        enable_php_remote_debugging: A boolean indicating whether the PHP
            interpreter should be started with XDebug remote debugging enabled.
        python_config: A runtime_config_pb2.PythonConfig instance containing
            Python runtime-specific configuration. If None then defaults are
            used.
        cloud_sql_config: A runtime_config_pb2.CloudSQL instance containing the
            required configuration for local Google Cloud SQL development. If None
            then Cloud SQL will not be available.
        default_version_port: An int containing the port of the default version.
        port_registry: A dispatcher.PortRegistry used to provide the Dispatcher
            with a mapping of port to Module and Instance.
        request_data: A wsgi_request_info.WSGIRequestInfo that will be provided
            with request information for use by API stubs.
        dispatcher: A Dispatcher instance that can be used to make HTTP requests.
        use_mtime_file_watcher: A bool containing whether to use mtime polling to
            monitor file changes even if other options are available on the
            current platform.
        allow_skipped_files: If True then all files in the application's directory
            are readable, even if they appear in a static handler or "skip_files"
            directive.
      """
      super(InteractiveCommandModule, self).__init__(
          module_configuration,
          host,
          balanced_port,
          api_port,
          auth_domain,
          runtime_stderr_loglevel,
          php_executable_path,
          enable_php_remote_debugging,
          python_config,
          cloud_sql_config,
          default_version_port,
          port_registry,
          request_data,
          dispatcher,
          max_instances=1,
          use_mtime_file_watcher=use_mtime_file_watcher,
          automatic_restarts=True,
          allow_skipped_files=allow_skipped_files)
      # Use a single instance so that state is consistent across requests.
      self._inst_lock = threading.Lock()
      self._inst = None
  
    @property
    def balanced_port(self):
      """The port that the balanced HTTP server for the Module is listening on.
  
      The InteractiveCommandModule does not actually listen on this port but it is
      used when constructing the "SERVER_PORT" in the WSGI-environment.
      """
      return self._balanced_port
  
    def quit(self):
      """Stops the InteractiveCommandModule."""
      if self._inst:
        self._inst.quit(force=True)
        self._inst = None
  
    def _handle_script_request(self,
                               environ,
                               start_response,
                               url_map,
                               match,
                               request_id,
                               inst=None,
                               request_type=instance.INTERACTIVE_REQUEST):
      """Handles a interactive request by forwarding it to the managed Instance.
  
      Args:
        environ: An environ dict for the request as defined in PEP-333.
        start_response: A function with semantics defined in PEP-333.
        url_map: An appinfo.URLMap instance containing the configuration for the
            handler that matched.
        match: A re.MatchObject containing the result of the matched URL pattern.
        request_id: A unique string id associated with the request.
        inst: The instance.Instance to send the request to.
        request_type: The type of the request. See instance.*_REQUEST module
            constants. This must be instance.INTERACTIVE_REQUEST.
  
      Returns:
        An iterable over strings containing the body of the HTTP response.
      """
      assert inst is None
      assert request_type == instance.INTERACTIVE_REQUEST
  
      start_time = time.time()
      timeout_time = start_time + self._MAX_REQUEST_WAIT_TIME
  
      while time.time() < timeout_time:
        new_instance = False
        with self._inst_lock:
          if not self._inst:
            self._inst = self._instance_factory.new_instance(
                AutoScalingModule.generate_instance_id(),
                expect_ready_request=False)
            new_instance = True
          inst = self._inst
  
        if new_instance:
          self._inst.start()
  
        try:
          return inst.handle(environ, start_response, url_map, match,
                             request_id, request_type)
        except instance.CannotAcceptRequests:
          inst.wait(timeout_time)
        except Exception:
          # If the instance is restarted while handling a request then the
          # exception raises is unpredictable.
          if inst != self._inst:
            start_response('503 Service Unavailable', [])
            return ['Instance was restarted while executing command']
          logging.exception('Unexpected exception handling command: %r', environ)
          raise
      else:
        start_response('503 Service Unavailable', [])
        return ['The command timed-out while waiting for another one to complete']
  
    def restart(self):
      """Restarts the the module."""
      with self._inst_lock:
        if self._inst:
          self._inst.quit(force=True)
          self._inst = None
  
    def send_interactive_command(self, command):
      """Sends an interactive command to the module.
  
      Args:
        command: The command to send e.g. "print 5+5".
  
      Returns:
        A string representing the result of the command e.g. "10\n".
  
      Raises:
        InteractiveCommandError: if the command failed for any reason.
      """
      start_response = start_response_utils.CapturingStartResponse()
  
      # 192.0.2.0 is an example address defined in RFC 5737.
      environ = self.build_request_environ(
          'POST', '/', [], command, '192.0.2.0', self.balanced_port)
  
      try:
        response = self._handle_request(
            environ,
            start_response,
            request_type=instance.INTERACTIVE_REQUEST)
      except Exception as e:
        raise InteractiveCommandError('Unexpected command failure: ', str(e))
  
      if start_response.status != '200 OK':
        raise InteractiveCommandError(start_response.merged_response(response))
  
      return start_response.merged_response(response)
  
files: AppServer/google/appengine/tools/devappserver2/module.py
files: #!/usr/bin/env python
  #
  # Copyright 2007 Google Inc.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  #
  """Manage the lifecycle of runtime processes and dispatch requests to them."""
  
  
  import collections
  import cStringIO
  import functools
  import httplib
  import logging
  import math
  import os
  import os.path
  import random
  import re
  import string
  import threading
  import time
  import urllib
  import urlparse
  import wsgiref.headers
  
  from google.appengine.api import api_base_pb
  from google.appengine.api import apiproxy_stub_map
  from google.appengine.api import appinfo
  from google.appengine.api import request_info
  from google.appengine.api.logservice import log_service_pb
  from google.appengine.tools.devappserver2 import application_configuration
  from google.appengine.tools.devappserver2 import blob_image
  from google.appengine.tools.devappserver2 import blob_upload
  from google.appengine.tools.devappserver2 import channel
  from google.appengine.tools.devappserver2 import constants
  
  from google.appengine.tools.devappserver2 import endpoints
  from google.appengine.tools.devappserver2 import errors
  from google.appengine.tools.devappserver2 import file_watcher
  from google.appengine.tools.devappserver2 import go_runtime
  from google.appengine.tools.devappserver2 import http_runtime_constants
  from google.appengine.tools.devappserver2 import instance
  from google.appengine.tools.devappserver2 import login
  from google.appengine.tools.devappserver2 import php_runtime
  from google.appengine.tools.devappserver2 import python_runtime
  from google.appengine.tools.devappserver2 import request_rewriter
  from google.appengine.tools.devappserver2 import runtime_config_pb2
  from google.appengine.tools.devappserver2 import start_response_utils
  from google.appengine.tools.devappserver2 import static_files_handler
  from google.appengine.tools.devappserver2 import thread_executor
  from google.appengine.tools.devappserver2 import url_handler
  from google.appengine.tools.devappserver2 import util
  from google.appengine.tools.devappserver2 import wsgi_handler
  from google.appengine.tools.devappserver2 import wsgi_server
  
  
  _LOWER_HEX_DIGITS = string.hexdigits.lower()
  _UPPER_HEX_DIGITS = string.hexdigits.upper()
  _REQUEST_ID_HASH_LENGTH = 8
  
  _THREAD_POOL = thread_executor.ThreadExecutor()
  _RESTART_INSTANCES_CONFIG_CHANGES = frozenset(
      [application_configuration.NORMALIZED_LIBRARIES_CHANGED,
       application_configuration.SKIP_FILES_CHANGED,
       application_configuration.NOBUILD_FILES_CHANGED,
       # The server must be restarted when the handlers change because files
       # appearing in static content handlers make them unavailable to the
       # runtime.
       application_configuration.HANDLERS_CHANGED,
       application_configuration.ENV_VARIABLES_CHANGED])
  
  _REQUEST_LOGGING_BLACKLIST_RE = re.compile(
      r'^/_ah/(?:channel/(?:dev|jsapi)|img|login|upload)')
  
  # Fake arguments for _handle_script_request for request types that don't use
  # user-specified handlers.
  _EMPTY_MATCH = re.match('', '')
  _DUMMY_URLMAP = appinfo.URLMap(script='/')
  _SHUTDOWN_TIMEOUT = 30
  
  
  def _static_files_regex_from_handlers(handlers):
    patterns = []
    for url_map in handlers:
      handler_type = url_map.GetHandlerType()
      if url_map.application_readable:
        continue
      if handler_type == appinfo.STATIC_FILES:
        patterns.append(r'(%s)' % url_map.upload)
      elif handler_type == appinfo.STATIC_DIR:
        patterns.append('(%s%s%s)' % (url_map.static_dir.rstrip(os.path.sep),
                                      re.escape(os.path.sep), r'.*'))
    return r'^%s$' % '|'.join(patterns)
  
  
  class InteractiveCommandError(errors.Error):
    pass
  
  
  class _ScriptHandler(url_handler.UserConfiguredURLHandler):
    """A URL handler that will cause the request to be dispatched to an instance.
  
    This handler is special in that it does not have a working handle() method
    since the Module's dispatch logic is used to select the appropriate Instance.
    """
  
    def __init__(self, url_map):
      """Initializer for _ScriptHandler.
  
      Args:
        url_map: An appinfo.URLMap instance containing the configuration for this
            handler.
      """
      try:
        url_pattern = re.compile('%s$' % url_map.url)
      except re.error, e:
        raise errors.InvalidAppConfigError(
            'invalid url %r in script handler: %s' % (url_map.url, e))
  
      super(_ScriptHandler, self).__init__(url_map, url_pattern)
      self.url_map = url_map
  
    def handle(self, match, environ, start_response):
      """This is a dummy method that should never be called."""
      raise NotImplementedError()
  
  
  class Module(object):
    """The abstract base for all instance pool implementations."""
  
    _RUNTIME_INSTANCE_FACTORIES = {
  
        'go': go_runtime.GoRuntimeInstanceFactory,
        'php': php_runtime.PHPRuntimeInstanceFactory,
        'python': python_runtime.PythonRuntimeInstanceFactory,
        'python27': python_runtime.PythonRuntimeInstanceFactory,
    }
  
    def _create_instance_factory(self,
                                 module_configuration):
      """Create an instance.InstanceFactory.
  
      Args:
        module_configuration: An application_configuration.ModuleConfiguration
            instance storing the configuration data for a module.
  
      Returns:
        A instance.InstanceFactory subclass that can be used to create instances
        with the provided configuration.
  
      Raises:
        RuntimeError: if the configuration specifies an unknown runtime.
      """
      # TODO: a bad runtime should be caught before we get here.
      if module_configuration.runtime not in self._RUNTIME_INSTANCE_FACTORIES:
        raise RuntimeError(
            'Unknown runtime %r; supported runtimes are %s.' %
            (module_configuration.runtime,
             ', '.join(
                 sorted(repr(k) for k in self._RUNTIME_INSTANCE_FACTORIES))))
      instance_factory = self._RUNTIME_INSTANCE_FACTORIES[
          module_configuration.runtime]
      return instance_factory(
          request_data=self._request_data,
          runtime_config_getter=self._get_runtime_config,
          module_configuration=module_configuration)
  
    def _create_url_handlers(self):
      """Constructs URLHandlers based on the module configuration.
  
      Returns:
        A list of url_handler.URLHandlers corresponding that can react as
        described in the given configuration.
      """
      handlers = []
      # Add special URL handlers (taking precedence over user-defined handlers)
      url_pattern = '/%s$' % login.LOGIN_URL_RELATIVE
      handlers.append(wsgi_handler.WSGIHandler(login.application,
                                               url_pattern))
      url_pattern = '/%s' % blob_upload.UPLOAD_URL_PATH
      # The blobstore upload handler forwards successful requests back to self
      handlers.append(
          wsgi_handler.WSGIHandler(blob_upload.Application(self), url_pattern))
  
      url_pattern = '/%s' % blob_image.BLOBIMAGE_URL_PATTERN
      handlers.append(
          wsgi_handler.WSGIHandler(blob_image.Application(), url_pattern))
  
      url_pattern = '/%s' % channel.CHANNEL_URL_PATTERN
      handlers.append(
          wsgi_handler.WSGIHandler(channel.application, url_pattern))
  
      url_pattern = '/%s' % endpoints.API_SERVING_PATTERN
      handlers.append(
          wsgi_handler.WSGIHandler(
              endpoints.EndpointsDispatcher(self._dispatcher), url_pattern))
  
      found_start_handler = False
      found_warmup_handler = False
      # Add user-defined URL handlers
      for url_map in self._module_configuration.handlers:
        handler_type = url_map.GetHandlerType()
        if handler_type == appinfo.HANDLER_SCRIPT:
          handlers.append(_ScriptHandler(url_map))
          if not found_start_handler and re.match('%s$' % url_map.url,
                                                  '/_ah/start'):
            found_start_handler = True
          if not found_warmup_handler and re.match('%s$' % url_map.url,
                                                   '/_ah/warmup'):
            found_warmup_handler = True
        elif handler_type == appinfo.STATIC_FILES:
          handlers.append(
              static_files_handler.StaticFilesHandler(
                  self._module_configuration.application_root,
                  url_map))
        elif handler_type == appinfo.STATIC_DIR:
          handlers.append(
              static_files_handler.StaticDirHandler(
                  self._module_configuration.application_root,
                  url_map))
        else:
          assert 0, 'unexpected handler %r for %r' % (handler_type, url_map)
      # Add a handler for /_ah/start if no script handler matches.
      if not found_start_handler:
        handlers.insert(0, _ScriptHandler(self._instance_factory.START_URL_MAP))
      # Add a handler for /_ah/warmup if no script handler matches and warmup is
      # enabled.
      if (not found_warmup_handler and
          'warmup' in (self._module_configuration.inbound_services or [])):
        handlers.insert(0, _ScriptHandler(self._instance_factory.WARMUP_URL_MAP))
      return handlers
  
    def _get_runtime_config(self):
      """Returns the configuration for the runtime.
  
      Returns:
        A runtime_config_pb2.Config instance representing the configuration to be
        passed to an instance. NOTE: This does *not* include the instance_id
        field, which must be populated elsewhere.
      """
      runtime_config = runtime_config_pb2.Config()
      runtime_config.app_id = self._module_configuration.application
      runtime_config.version_id = self._module_configuration.version_id
      runtime_config.threadsafe = self._module_configuration.threadsafe or False
      runtime_config.application_root = (
          self._module_configuration.application_root)
      if not self._allow_skipped_files:
        runtime_config.skip_files = str(self._module_configuration.skip_files)
        runtime_config.static_files = _static_files_regex_from_handlers(
            self._module_configuration.handlers)
      runtime_config.api_port = self._api_port
      runtime_config.stderr_log_level = self._runtime_stderr_loglevel
      runtime_config.datacenter = 'us1'
      runtime_config.auth_domain = self._auth_domain
  
      for library in self._module_configuration.normalized_libraries:
        runtime_config.libraries.add(name=library.name, version=library.version)
  
      for key, value in (self._module_configuration.env_variables or {}).items():
        runtime_config.environ.add(key=str(key), value=str(value))
  
      if self._cloud_sql_config:
        runtime_config.cloud_sql_config.CopyFrom(self._cloud_sql_config)
  
      if self._module_configuration.runtime == 'php':
        if self._php_executable_path:
          runtime_config.php_config.php_executable_path = (
              self._php_executable_path)
        runtime_config.php_config.enable_debugger = (
            self._enable_php_remote_debugging)
      if (self._python_config and
          self._module_configuration.runtime.startswith('python')):
        runtime_config.python_config.CopyFrom(self._python_config)
      return runtime_config
  
    def _maybe_restart_instances(self, config_changed, file_changed):
      """Restarts instances. May avoid some restarts depending on policy.
  
      One of config_changed or file_changed must be True.
  
      Args:
        config_changed: True if the configuration for the application has changed.
        file_changed: True if any file relevant to the application has changed.
      """
      if not config_changed and not file_changed:
        return
  
      logging.debug('Restarting instances.')
      policy = self._instance_factory.FILE_CHANGE_INSTANCE_RESTART_POLICY
      assert policy is not None, 'FILE_CHANGE_INSTANCE_RESTART_POLICY not set'
  
      with self._condition:
        instances_to_quit = set()
        for inst in self._instances:
          if (config_changed or
              (policy == instance.ALWAYS) or
              (policy == instance.AFTER_FIRST_REQUEST and inst.total_requests)):
            instances_to_quit.add(inst)
        self._instances -= instances_to_quit
  
      for inst in instances_to_quit:
        inst.quit(allow_async=True)
  
    def _handle_changes(self):
      """Handle file or configuration changes."""
      # Always check for config and file changes because checking also clears
      # pending changes.
      config_changes = self._module_configuration.check_for_updates()
      has_file_changes = self._watcher.has_changes()
  
      if application_configuration.HANDLERS_CHANGED in config_changes:
        handlers = self._create_url_handlers()
        with self._handler_lock:
          self._handlers = handlers
  
      if has_file_changes:
        self._instance_factory.files_changed()
  
      if config_changes & _RESTART_INSTANCES_CONFIG_CHANGES:
        self._instance_factory.configuration_changed(config_changes)
  
      self._maybe_restart_instances(
          config_changed=bool(config_changes & _RESTART_INSTANCES_CONFIG_CHANGES),
          file_changed=has_file_changes)
  
    def __init__(self,
                 module_configuration,
                 host,
                 balanced_port,
                 api_port,
                 auth_domain,
                 runtime_stderr_loglevel,
                 php_executable_path,
                 enable_php_remote_debugging,
                 python_config,
                 cloud_sql_config,
                 default_version_port,
                 port_registry,
                 request_data,
                 dispatcher,
                 max_instances,
                 use_mtime_file_watcher,
                 automatic_restarts,
                 allow_skipped_files):
      """Initializer for Module.
  
      Args:
        module_configuration: An application_configuration.ModuleConfiguration
            instance storing the configuration data for a module.
        host: A string containing the host that any HTTP servers should bind to
            e.g. "localhost".
        balanced_port: An int specifying the port where the balanced module for
            the pool should listen.
        api_port: The port that APIModule listens for RPC requests on.
        auth_domain: A string containing the auth domain to set in the environment
            variables.
        runtime_stderr_loglevel: An int reprenting the minimum logging level at
            which runtime log messages should be written to stderr. See
            devappserver2.py for possible values.
        php_executable_path: A string containing the path to PHP execution e.g.
            "/usr/bin/php-cgi".
        enable_php_remote_debugging: A boolean indicating whether the PHP
            interpreter should be started with XDebug remote debugging enabled.
        python_config: A runtime_config_pb2.PythonConfig instance containing
            Python runtime-specific configuration. If None then defaults are
            used.
        cloud_sql_config: A runtime_config_pb2.CloudSQL instance containing the
            required configuration for local Google Cloud SQL development. If None
            then Cloud SQL will not be available.
        default_version_port: An int containing the port of the default version.
        port_registry: A dispatcher.PortRegistry used to provide the Dispatcher
            with a mapping of port to Module and Instance.
        request_data: A wsgi_request_info.WSGIRequestInfo that will be provided
            with request information for use by API stubs.
        dispatcher: A Dispatcher instance that can be used to make HTTP requests.
        max_instances: The maximum number of instances to create for this module.
            If None then there is no limit on the number of created instances.
        use_mtime_file_watcher: A bool containing whether to use mtime polling to
            monitor file changes even if other options are available on the
            current platform.
        automatic_restarts: If True then instances will be restarted when a
            file or configuration change that effects them is detected.
        allow_skipped_files: If True then all files in the application's directory
            are readable, even if they appear in a static handler or "skip_files"
            directive.
      """
      self._module_configuration = module_configuration
      self._name = module_configuration.module_name
      self._host = host
      self._api_port = api_port
      self._auth_domain = auth_domain
      self._runtime_stderr_loglevel = runtime_stderr_loglevel
      self._balanced_port = balanced_port
      self._php_executable_path = php_executable_path
      self._enable_php_remote_debugging = enable_php_remote_debugging
      self._python_config = python_config
      self._cloud_sql_config = cloud_sql_config
      self._request_data = request_data
      # _create_instance_factory() transitively calls _get_runtime_config, which
      # uses self._allow_skipped_files.
      self._allow_skipped_files = allow_skipped_files
      self._instance_factory = self._create_instance_factory(
          self._module_configuration)
      self._dispatcher = dispatcher
      self._max_instances = max_instances
      self._automatic_restarts = automatic_restarts
      self._use_mtime_file_watcher = use_mtime_file_watcher
      if self._automatic_restarts:
        self._watcher = file_watcher.get_file_watcher(
            [self._module_configuration.application_root] +
            self._instance_factory.get_restart_directories(),
            self._use_mtime_file_watcher)
      else:
        self._watcher = None
      self._handler_lock = threading.Lock()
      self._handlers = self._create_url_handlers()
      self._default_version_port = default_version_port
      self._port_registry = port_registry
  
      self._balanced_module = wsgi_server.WsgiServer(
          (self._host, self._balanced_port), self)
      self._quit_event = threading.Event()  # Set when quit() has been called.
  
      # AppScale: Keeps track of active requests in case of shutdown.
  
      # Indicates that the instance should refuse future requests.
      self.sigterm_sent = False
  
      # Handles request count and sigterm flag mutations.
      self.graceful_shutdown_lock = threading.Lock()
  
      # Keeps track of how many requests the instance is serving.
      self.request_count = 0
  
      # End AppScale.
  
    @property
    def name(self):
      """The name of the module, as defined in app.yaml.
  
      This value will be constant for the lifetime of the module even in the
      module configuration changes.
      """
      return self._name
  
    @property
    def ready(self):
      """The module is ready to handle HTTP requests."""
      return self._balanced_module.ready
  
    @property
    def balanced_port(self):
      """The port that the balanced HTTP server for the Module is listening on."""
      assert self._balanced_module.ready, 'balanced module not running'
      return self._balanced_module.port
  
    @property
    def host(self):
      """The host that the HTTP server(s) for this Module is listening on."""
      return self._host
  
    @property
    def balanced_address(self):
      """The address of the balanced HTTP server e.g. "localhost:8080"."""
      if self.balanced_port != 80:
        return '%s:%s' % (self.host, self.balanced_port)
      else:
        return self.host
  
    @property
    def max_instance_concurrent_requests(self):
      """The number of concurrent requests that each Instance can handle."""
      return self._instance_factory.max_concurrent_requests
  
    @property
    def module_configuration(self):
      """The application_configuration.ModuleConfiguration for this module."""
      return self._module_configuration
  
    @property
    def supports_interactive_commands(self):
      """True if the module can evaluate arbitrary code and return the result."""
      return self._instance_factory.SUPPORTS_INTERACTIVE_REQUESTS
  
    def _handle_script_request(self,
                               environ,
                               start_response,
                               url_map,
                               match,
                               inst=None):
      """Handles a HTTP request that has matched a script handler.
  
      Args:
        environ: An environ dict for the request as defined in PEP-333.
        start_response: A function with semantics defined in PEP-333.
        url_map: An appinfo.URLMap instance containing the configuration for the
            handler that matched.
        match: A re.MatchObject containing the result of the matched URL pattern.
        inst: The Instance to send the request to. If None then an appropriate
            Instance will be chosen.
  
      Returns:
        An iterable over strings containing the body of the HTTP response.
      """
      raise NotImplementedError()
  
    def _no_handler_for_request(self, environ, start_response, request_id):
      """Handle a HTTP request that does not match any user-defined handlers."""
      self._insert_log_message('No handlers matched this URL.', 2, request_id)
      start_response('404 Not Found', [('Content-Type', 'text/plain')])
      return ['The url "%s" does not match any handlers.' % environ['PATH_INFO']]
  
    def _error_response(self, environ, start_response, status):
      start_response('%d %s' % (status, httplib.responses[status]), [])
      return []
  
    # AppScale: Check if the instance should be shutting down before handling
    # request.
    def _handle_request(self, environ, start_response, **kwargs):
      """ A _handle_request wrapper that keeps track of active requests.
  
      Args:
        environ: An environ dict for the request as defined in PEP-333.
        start_response: A function with semantics defined in PEP-333.
      Returns:
        An iterable over strings containing the body of the HTTP response.
      """
      with self.graceful_shutdown_lock:
        if self.sigterm_sent:
          start_response('503 Service Unavailable',
                         [('Content-Type', 'text/plain')])
          return ['This instance is shutting down']
  
        self.request_count += 1
  
      try:
        return self._handle_request_impl(environ, start_response, **kwargs)
      finally:
        with self.graceful_shutdown_lock:
          self.request_count -= 1
  
    # End AppScale.
  
    def _handle_request_impl(self, environ, start_response, inst=None,
                             request_type=instance.NORMAL_REQUEST):
      """Handles a HTTP request.
  
      Args:
        environ: An environ dict for the request as defined in PEP-333.
        start_response: A function with semantics defined in PEP-333.
        inst: The Instance to send the request to. If None then an appropriate
            Instance will be chosen. Setting inst is not meaningful if the
            request does not match a "script" handler.
        request_type: The type of the request. See instance.*_REQUEST module
            constants.
  
      Returns:
        An iterable over strings containing the body of the HTTP response.
      """
      try:
        environ['SERVER_PORT'] = environ['HTTP_HOST'].split(':')[1]
      except IndexError:
        scheme = environ['HTTP_X_FORWARDED_PROTO']
        if scheme == 'http':
          environ['SERVER_PORT'] = 80
        else:
          environ['SERVER_PORT'] = 443
  
      if 'HTTP_HOST' in environ:
        environ['SERVER_NAME'] = environ['HTTP_HOST'].split(':', 1)[0]
      environ['DEFAULT_VERSION_HOSTNAME'] = '%s:%s' % (
        environ['SERVER_NAME'], environ['SERVER_PORT'])
      with self._request_data.request(
          environ,
          self._module_configuration) as request_id:
        should_log_request = not _REQUEST_LOGGING_BLACKLIST_RE.match(
            environ['PATH_INFO'])
        environ['REQUEST_ID_HASH'] = self.generate_request_id_hash()
        if should_log_request:
          environ['REQUEST_LOG_ID'] = self.generate_request_log_id()
          if 'HTTP_HOST' in environ:
            hostname = environ['HTTP_HOST']
          elif environ['SERVER_PORT'] == '80':
            hostname = environ['SERVER_NAME']
          else:
            hostname = '%s:%s' % (environ['SERVER_NAME'], environ['SERVER_PORT'])
  
          if environ.get('QUERY_STRING'):
            resource = '%s?%s' % (urllib.quote(environ['PATH_INFO']),
                                  environ['QUERY_STRING'])
          else:
            resource = urllib.quote(environ['PATH_INFO'])
          email, _, _ = login.get_user_info(environ.get('HTTP_COOKIE', ''))
          method = environ.get('REQUEST_METHOD', 'GET')
          http_version = environ.get('SERVER_PROTOCOL', 'HTTP/1.0')
  
          logservice = apiproxy_stub_map.apiproxy.GetStub('logservice')
          logservice.start_request(
              request_id=request_id,
              user_request_id=environ['REQUEST_LOG_ID'],
              ip=environ.get('REMOTE_ADDR', ''),
              app_id=self._module_configuration.application,
              version_id=self._module_configuration.version_id,
              nickname=email.split('@', 1)[0],
              user_agent=environ.get('HTTP_USER_AGENT', ''),
              host=hostname,
              method=method,
              resource=resource,
              http_version=http_version)
  
        def wrapped_start_response(status, response_headers, exc_info=None):
          response_headers.append(('Server',
                                   http_runtime_constants.SERVER_SOFTWARE))
          if should_log_request:
            headers = wsgiref.headers.Headers(response_headers)
            status_code = int(status.split(' ', 1)[0])
            content_length = int(headers.get('Content-Length', 0))
            logservice.end_request(request_id, status_code, content_length)
            logging.info('%(module_name)s: '
                         '"%(method)s %(resource)s %(http_version)s" '
                         '%(status)d %(content_length)s',
                         {'module_name': self.name,
                          'method': method,
                          'resource': resource,
                          'http_version': http_version,
                          'status': status_code,
                          'content_length': content_length or '-'})
          return start_response(status, response_headers, exc_info)
  
        if (environ['REQUEST_METHOD'] in ('GET', 'HEAD', 'TRACE') and
            int(environ.get('CONTENT_LENGTH') or '0') != 0):
          # CONTENT_LENGTH may be empty or absent.
          wrapped_start_response('400 Bad Request', [])
          return ['"%s" requests may not contain bodies.' %
                  environ['REQUEST_METHOD']]
  
        with self._handler_lock:
          handlers = self._handlers
  
        try:
          request_url = environ['PATH_INFO']
          if request_type in (instance.BACKGROUND_REQUEST,
                              instance.INTERACTIVE_REQUEST,
                              instance.SHUTDOWN_REQUEST):
            app = functools.partial(self._handle_script_request,
                                    url_map=_DUMMY_URLMAP,
                                    match=_EMPTY_MATCH,
                                    request_id=request_id,
                                    inst=inst,
                                    request_type=request_type)
            return request_rewriter.frontend_rewriter_middleware(app)(
                environ, wrapped_start_response)
          for handler in handlers:
            match = handler.match(request_url)
            if match:
              auth_failure = handler.handle_authorization(environ,
                                                          wrapped_start_response)
              if auth_failure is not None:
                return auth_failure
  
              if isinstance(handler, _ScriptHandler):
                app = functools.partial(self._handle_script_request,
                                        url_map=handler.url_map,
                                        match=match,
                                        request_id=request_id,
                                        inst=inst,
                                        request_type=request_type)
                return request_rewriter.frontend_rewriter_middleware(app)(
                    environ, wrapped_start_response)
              else:
                return handler.handle(match, environ, wrapped_start_response)
          return self._no_handler_for_request(environ, wrapped_start_response,
                                              request_id)
        except StandardError, e:
          logging.exception('Request to %r failed', request_url)
          wrapped_start_response('500 Internal Server Error', [], e)
          return []
  
    def _async_shutdown_instance(self, inst, port):
      _THREAD_POOL.submit(self._shutdown_instance, inst, port)
  
    def _shutdown_instance(self, inst, port):
      force_shutdown_time = time.time() + _SHUTDOWN_TIMEOUT
      try:
        environ = self.build_request_environ(
            'GET', '/_ah/stop', [], '', '0.1.0.3', port, fake_login=True)
        self._handle_request(environ,
                             start_response_utils.null_start_response,
                             inst=inst,
                             request_type=instance.SHUTDOWN_REQUEST)
        logging.debug('Sent shutdown request: %s', inst)
      except:
        logging.exception('Internal error while handling shutdown request.')
      finally:
        time_to_wait = force_shutdown_time - time.time()
        self._quit_event.wait(time_to_wait)
        inst.quit(force=True)
  
    def _insert_log_message(self, message, level, request_id):
      logs_group = log_service_pb.UserAppLogGroup()
      log_line = logs_group.add_log_line()
      log_line.set_timestamp_usec(int(time.time() * 1e6))
      log_line.set_level(level)
      log_line.set_message(message)
      request = log_service_pb.FlushRequest()
      request.set_logs(logs_group.Encode())
      response = api_base_pb.VoidProto()
      logservice = apiproxy_stub_map.apiproxy.GetStub('logservice')
      logservice._Dynamic_Flush(request, response, request_id)
  
    @staticmethod
    def generate_request_log_id():
      """Generate a random REQUEST_LOG_ID.
  
      Returns:
        A string suitable for use as a REQUEST_LOG_ID. The returned string is
        variable length to emulate the the production values, which encapsulate
        the application id, version and some log state.
      """
      return ''.join(random.choice(_LOWER_HEX_DIGITS)
                     for _ in range(random.randrange(30, 100)))
  
    @staticmethod
    def generate_request_id_hash():
      """Generate a random REQUEST_ID_HASH."""
      return ''.join(random.choice(_UPPER_HEX_DIGITS)
                     for _ in range(_REQUEST_ID_HASH_LENGTH))
  
    def set_num_instances(self, instances):
      """Sets the number of instances for this module to run.
  
      Args:
        instances: An int containing the number of instances to run.
      """
      raise request_info.NotSupportedWithAutoScalingError()
  
    def get_num_instances(self):
      """Returns the number of instances for this module to run."""
      raise request_info.NotSupportedWithAutoScalingError()
  
    def suspend(self):
      """Stops the module from serving requests."""
      raise request_info.NotSupportedWithAutoScalingError()
  
    def resume(self):
      """Restarts the module."""
      raise request_info.NotSupportedWithAutoScalingError()
  
    def get_instance_address(self, instance_id):
      """Returns the address of the HTTP server for an instance."""
      return '%s:%s' % (self.host, self.get_instance_port(instance_id))
  
    def get_instance_port(self, instance_id):
      """Returns the port of the HTTP server for an instance."""
      raise request_info.NotSupportedWithAutoScalingError()
  
    def get_instance(self, instance_id):
      """Returns the instance with the provided instance ID."""
      raise request_info.NotSupportedWithAutoScalingError()
  
    @property
    def supports_individually_addressable_instances(self):
      return False
  
    def create_interactive_command_module(self):
      """Returns a InteractiveCommandModule that can be sent user commands."""
      if self._instance_factory.SUPPORTS_INTERACTIVE_REQUESTS:
        return InteractiveCommandModule(self._module_configuration,
                                        self._host,
                                        self._balanced_port,
                                        self._api_port,
                                        self._auth_domain,
                                        self._runtime_stderr_loglevel,
                                        self._php_executable_path,
                                        self._enable_php_remote_debugging,
                                        self._python_config,
                                        self._cloud_sql_config,
                                        self._default_version_port,
                                        self._port_registry,
                                        self._request_data,
                                        self._dispatcher,
                                        self._use_mtime_file_watcher,
                                        self._allow_skipped_files)
      else:
        raise NotImplementedError('runtime does not support interactive commands')
  
    def build_request_environ(self, method, relative_url, headers, body,
                              source_ip, port, fake_login=False):
      if isinstance(body, unicode):
        body = body.encode('ascii')
  
      url = urlparse.urlsplit(relative_url)
      if port != 80:
        host = '%s:%s' % (self.host, port)
      else:
        host = self.host
      environ = {constants.FAKE_IS_ADMIN_HEADER: '1',
                 'CONTENT_LENGTH': str(len(body)),
                 'PATH_INFO': url.path,
                 'QUERY_STRING': url.query,
                 'REQUEST_METHOD': method,
                 'REMOTE_ADDR': source_ip,
                 'SERVER_NAME': self.host,
                 'SERVER_PORT': str(port),
                 'SERVER_PROTOCOL': 'HTTP/1.1',
                 'wsgi.version': (1, 0),
                 'wsgi.url_scheme': 'http',
                 'wsgi.errors': cStringIO.StringIO(),
                 'wsgi.multithread': True,
                 'wsgi.multiprocess': True,
                 'wsgi.input': cStringIO.StringIO(body)}
      if fake_login:
        environ[constants.FAKE_LOGGED_IN_HEADER] = '1'
      util.put_headers_in_environ(headers, environ)
      environ['HTTP_HOST'] = host
      return environ
  
  
  class AutoScalingModule(Module):
    """A pool of instances that is autoscaled based on traffic."""
  
    # The minimum number of seconds to wait, after quitting an idle instance,
    # before quitting another idle instance.
    _MIN_SECONDS_BETWEEN_QUITS = 60
    # The time horizon to use when calculating the number of instances required
    # to serve the current level of traffic.
    _REQUIRED_INSTANCE_WINDOW_SECONDS = 60
  
    _DEFAULT_AUTOMATIC_SCALING = appinfo.AutomaticScaling(
        min_pending_latency='0.1s',
        max_pending_latency='0.5s',
        min_idle_instances=1,
        max_idle_instances=1000)
  
    @staticmethod
    def _parse_pending_latency(timing):
      """Parse a pending latency string into a float of the value in seconds.
  
      Args:
        timing: A str of the form 1.0s or 1000ms.
  
      Returns:
        A float representation of the value in seconds.
      """
      if timing.endswith('ms'):
        return float(timing[:-2]) / 1000
      else:
        return float(timing[:-1])
  
    @classmethod
    def _populate_default_automatic_scaling(cls, automatic_scaling):
      for attribute in automatic_scaling.ATTRIBUTES:
        if getattr(automatic_scaling, attribute) in ('automatic', None):
          setattr(automatic_scaling, attribute,
                  getattr(cls._DEFAULT_AUTOMATIC_SCALING, attribute))
  
    def _process_automatic_scaling(self, automatic_scaling):
      if automatic_scaling:
        self._populate_default_automatic_scaling(automatic_scaling)
      else:
        automatic_scaling = self._DEFAULT_AUTOMATIC_SCALING
      self._min_pending_latency = self._parse_pending_latency(
          automatic_scaling.min_pending_latency)
      self._max_pending_latency = self._parse_pending_latency(
          automatic_scaling.max_pending_latency)
      self._min_idle_instances = int(automatic_scaling.min_idle_instances)
      self._max_idle_instances = int(automatic_scaling.max_idle_instances)
  
    def __init__(self,
                 module_configuration,
                 host,
                 balanced_port,
                 api_port,
                 auth_domain,
                 runtime_stderr_loglevel,
                 php_executable_path,
                 enable_php_remote_debugging,
                 python_config,
                 cloud_sql_config,
                 default_version_port,
                 port_registry,
                 request_data,
                 dispatcher,
                 max_instances,
                 use_mtime_file_watcher,
                 automatic_restarts,
                 allow_skipped_files):
      """Initializer for AutoScalingModule.
  
      Args:
        module_configuration: An application_configuration.ModuleConfiguration
            instance storing the configuration data for a module.
        host: A string containing the host that any HTTP servers should bind to
            e.g. "localhost".
        balanced_port: An int specifying the port where the balanced module for
            the pool should listen.
        api_port: The port that APIServer listens for RPC requests on.
        auth_domain: A string containing the auth domain to set in the environment
            variables.
        runtime_stderr_loglevel: An int reprenting the minimum logging level at
            which runtime log messages should be written to stderr. See
            devappserver2.py for possible values.
        php_executable_path: A string containing the path to PHP execution e.g.
            "/usr/bin/php-cgi".
        enable_php_remote_debugging: A boolean indicating whether the PHP
            interpreter should be started with XDebug remote debugging enabled.
        python_config: A runtime_config_pb2.PythonConfig instance containing
            Python runtime-specific configuration. If None then defaults are
            used.
        cloud_sql_config: A runtime_config_pb2.CloudSQL instance containing the
            required configuration for local Google Cloud SQL development. If None
            then Cloud SQL will not be available.
        default_version_port: An int containing the port of the default version.
        port_registry: A dispatcher.PortRegistry used to provide the Dispatcher
            with a mapping of port to Module and Instance.
        request_data: A wsgi_request_info.WSGIRequestInfo that will be provided
            with request information for use by API stubs.
        dispatcher: A Dispatcher instance that can be used to make HTTP requests.
        max_instances: The maximum number of instances to create for this module.
            If None then there is no limit on the number of created instances.
        use_mtime_file_watcher: A bool containing whether to use mtime polling to
            monitor file changes even if other options are available on the
            current platform.
        automatic_restarts: If True then instances will be restarted when a
            file or configuration change that effects them is detected.
        allow_skipped_files: If True then all files in the application's directory
            are readable, even if they appear in a static handler or "skip_files"
            directive.
      """
      super(AutoScalingModule, self).__init__(module_configuration,
                                              host,
                                              balanced_port,
                                              api_port,
                                              auth_domain,
                                              runtime_stderr_loglevel,
                                              php_executable_path,
                                              enable_php_remote_debugging,
                                              python_config,
                                              cloud_sql_config,
                                              default_version_port,
                                              port_registry,
                                              request_data,
                                              dispatcher,
                                              max_instances,
                                              use_mtime_file_watcher,
                                              automatic_restarts,
                                              allow_skipped_files)
  
      self._process_automatic_scaling(
          self._module_configuration.automatic_scaling)
  
      self._instances = set()  # Protected by self._condition.
      # A deque containg (time, num_outstanding_instance_requests) 2-tuples.
      # This is used to track the maximum number of outstanding requests in a time
      # period. Protected by self._condition.
      self._outstanding_request_history = collections.deque()
      self._num_outstanding_instance_requests = 0  # Protected by self._condition.
      # The time when the last instance was quit in seconds since the epoch.
      self._last_instance_quit_time = 0  # Protected by self._condition.
  
      self._condition = threading.Condition()  # Protects instance state.
      self._instance_adjustment_thread = threading.Thread(
          target=self._loop_adjusting_instances)
  
    def start(self):
      """Start background management of the Module."""
      self._balanced_module.start()
      self._port_registry.add(self.balanced_port, self, None)
      if self._watcher:
        self._watcher.start()
      self._instance_adjustment_thread.start()
  
    def quit(self):
      """Stops the Module."""
      self._quit_event.set()
      self._instance_adjustment_thread.join()
      # The instance adjustment thread depends on the balanced module and the
      # watcher so wait for it exit before quitting them.
      if self._watcher:
        self._watcher.quit()
      self._balanced_module.quit()
      with self._condition:
        instances = self._instances
        self._instances = set()
        self._condition.notify_all()
      for inst in instances:
        inst.quit(force=True)
  
    @property
    def instances(self):
      """A set of all the instances currently in the Module."""
      with self._condition:
        return set(self._instances)
  
    @property
    def num_outstanding_instance_requests(self):
      """The number of requests that instances are currently handling."""
      with self._condition:
        return self._num_outstanding_instance_requests
  
    def _handle_instance_request(self,
                                 environ,
                                 start_response,
                                 url_map,
                                 match,
                                 request_id,
                                 inst,
                                 request_type):
      """Handles a request routed a particular Instance.
  
      Args:
        environ: An environ dict for the request as defined in PEP-333.
        start_response: A function with semantics defined in PEP-333.
        url_map: An appinfo.URLMap instance containing the configuration for the
            handler that matched.
        match: A re.MatchObject containing the result of the matched URL pattern.
        request_id: A unique string id associated with the request.
        inst: The instance.Instance to send the request to.
        request_type: The type of the request. See instance.*_REQUEST module
            constants.
  
      Returns:
        An iterable over strings containing the body of the HTTP response.
      """
      if request_type != instance.READY_REQUEST:
        with self._condition:
          self._num_outstanding_instance_requests += 1
          self._outstanding_request_history.append(
              (time.time(), self.num_outstanding_instance_requests))
      try:
        logging.debug('Dispatching request to %s', inst)
        return inst.handle(environ, start_response, url_map, match, request_id,
                           request_type)
      finally:
        with self._condition:
          if request_type != instance.READY_REQUEST:
            self._num_outstanding_instance_requests -= 1
          self._condition.notify()
  
    def _handle_script_request(self,
                               environ,
                               start_response,
                               url_map,
                               match,
                               request_id,
                               inst=None,
                               request_type=instance.NORMAL_REQUEST):
      """Handles a HTTP request that has matched a script handler.
  
      Args:
        environ: An environ dict for the request as defined in PEP-333.
        start_response: A function with semantics defined in PEP-333.
        url_map: An appinfo.URLMap instance containing the configuration for the
            handler that matched.
        match: A re.MatchObject containing the result of the matched URL pattern.
        request_id: A unique string id associated with the request.
        inst: The instance.Instance to send the request to. If None then an
            appropriate instance.Instance will be chosen.
        request_type: The type of the request. See instance.*_REQUEST module
            constants.
  
      Returns:
        An iterable over strings containing the body of the HTTP response.
      """
      if inst is not None:
        return self._handle_instance_request(
            environ, start_response, url_map, match, request_id, inst,
            request_type)
  
      with self._condition:
        self._num_outstanding_instance_requests += 1
        self._outstanding_request_history.append(
            (time.time(), self.num_outstanding_instance_requests))
  
      try:
        start_time = time.time()
        timeout_time = start_time + self._min_pending_latency
        # Loop until an instance is available to handle the request.
        while True:
          if self._quit_event.is_set():
            return self._error_response(environ, start_response, 404)
          inst = self._choose_instance(timeout_time)
          if not inst:
            inst = self._add_instance(permit_warmup=False)
            if not inst:
              # No instance is available nor can a new one be created, so loop
              # waiting for one to be free.
              timeout_time = time.time() + 0.2
              continue
  
          try:
            logging.debug('Dispatching request to %s after %0.4fs pending',
                          inst, time.time() - start_time)
            return inst.handle(environ,
                               start_response,
                               url_map,
                               match,
                               request_id,
                               request_type)
          except instance.CannotAcceptRequests:
            continue
      finally:
        with self._condition:
          self._num_outstanding_instance_requests -= 1
          self._condition.notify()
  
    def _add_instance(self, permit_warmup):
      """Creates and adds a new instance.Instance to the Module.
  
      Args:
        permit_warmup: If True then the new instance.Instance will be sent a new
            warmup request if it is configured to receive them.
  
      Returns:
        The newly created instance.Instance. Returns None if no new instance
        could be created because the maximum number of instances have already
        been created.
      """
      if self._max_instances is not None:
        with self._condition:
          if len(self._instances) >= self._max_instances:
            return None
  
      perform_warmup = permit_warmup and (
          'warmup' in (self._module_configuration.inbound_services or []))
  
      inst = self._instance_factory.new_instance(
          self.generate_instance_id(),
          expect_ready_request=perform_warmup)
  
      with self._condition:
        if self._quit_event.is_set():
          return None
        self._instances.add(inst)
  
      if not inst.start():
        return None
  
      if perform_warmup:
        self._async_warmup(inst)
      else:
        with self._condition:
          self._condition.notify(self.max_instance_concurrent_requests)
      logging.debug('Created instance: %s', inst)
      return inst
  
    @staticmethod
    def generate_instance_id():
      return ''.join(random.choice(_LOWER_HEX_DIGITS) for _ in range(36))
  
    def _warmup(self, inst):
      """Send a warmup request to the given instance."""
  
      try:
        environ = self.build_request_environ(
            'GET', '/_ah/warmup', [], '', '0.1.0.3', self.balanced_port,
            fake_login=True)
        self._handle_request(environ,
                             start_response_utils.null_start_response,
                             inst=inst,
                             request_type=instance.READY_REQUEST)
        with self._condition:
          self._condition.notify(self.max_instance_concurrent_requests)
      except:
        logging.exception('Internal error while handling warmup request.')
  
    def _async_warmup(self, inst):
      """Asynchronously send a markup request to the given Instance."""
      _THREAD_POOL.submit(self._warmup, inst)
  
    def _trim_outstanding_request_history(self):
      """Removes obsolete entries from _outstanding_request_history."""
      window_start = time.time() - self._REQUIRED_INSTANCE_WINDOW_SECONDS
      with self._condition:
        while self._outstanding_request_history:
          t, _ = self._outstanding_request_history[0]
          if t < window_start:
            self._outstanding_request_history.popleft()
          else:
            break
  
    def _get_num_required_instances(self):
      """Returns the number of Instances required to handle the request load."""
      with self._condition:
        self._trim_outstanding_request_history()
        if not self._outstanding_request_history:
          return 0
        else:
          peak_concurrent_requests = max(
              current_requests
              for (t, current_requests)
              in self._outstanding_request_history)
          return int(math.ceil(peak_concurrent_requests /
                               self.max_instance_concurrent_requests))
  
    def _split_instances(self):
      """Returns a 2-tuple representing the required and extra Instances.
  
      Returns:
        A 2-tuple of (required_instances, not_required_instances):
          required_instances: The set of the instance.Instances, in a state that
                              can handle requests, required to handle the current
                              request load.
          not_required_instances: The set of the Instances contained in this
                                  Module that not are not required.
      """
      with self._condition:
        num_required_instances = self._get_num_required_instances()
  
        available = [inst for inst in self._instances
                     if inst.can_accept_requests]
        available.sort(key=lambda inst: -inst.num_outstanding_requests)
  
        required = set(available[:num_required_instances])
        return required, self._instances - required
  
    def _choose_instance(self, timeout_time):
      """Returns the best Instance to handle a request or None if all are busy."""
      with self._condition:
        while time.time() < timeout_time:
          required_instances, not_required_instances = self._split_instances()
          if required_instances:
            # Pick the instance with the most remaining capacity to handle
            # requests.
            required_instances = sorted(
                required_instances,
                key=lambda inst: inst.remaining_request_capacity)
            if required_instances[-1].remaining_request_capacity:
              return required_instances[-1]
  
          available_instances = [inst for inst in not_required_instances
                                 if inst.remaining_request_capacity > 0 and
                                 inst.can_accept_requests]
          if available_instances:
            # Pick the instance with the *least* capacity to handle requests
            # to avoid using unnecessary idle instances.
            available_instances.sort(
                key=lambda instance: instance.num_outstanding_requests)
            return available_instances[-1]
          else:
            self._condition.wait(timeout_time - time.time())
      return None
  
    def _adjust_instances(self):
      """Creates new Instances or deletes idle Instances based on current load."""
      now = time.time()
      with self._condition:
        _, not_required_instances = self._split_instances()
  
      if len(not_required_instances) < self._min_idle_instances:
        self._add_instance(permit_warmup=True)
      elif (len(not_required_instances) > self._max_idle_instances and
            now >
            (self._last_instance_quit_time + self._MIN_SECONDS_BETWEEN_QUITS)):
        for inst in not_required_instances:
          if not inst.num_outstanding_requests:
            try:
              inst.quit()
            except instance.CannotQuitServingInstance:
              pass
            else:
              self._last_instance_quit_time = now
              logging.debug('Quit instance: %s', inst)
              with self._condition:
                self._instances.discard(inst)
                break
  
    def _loop_adjusting_instances(self):
      """Loops until the Module exits, reloading, adding or removing Instances."""
      while not self._quit_event.is_set():
        if self.ready:
          if self._automatic_restarts:
            self._handle_changes()
          self._adjust_instances()
        self._quit_event.wait(timeout=1)
  
    def __call__(self, environ, start_response):
      return self._handle_request(environ, start_response)
  
  
  class ManualScalingModule(Module):
    """A pool of instances that is manually-scaled."""
  
    _DEFAULT_MANUAL_SCALING = appinfo.ManualScaling(instances='1')
    _MAX_REQUEST_WAIT_TIME = 10
  
    @classmethod
    def _populate_default_manual_scaling(cls, manual_scaling):
      for attribute in manual_scaling.ATTRIBUTES:
        if getattr(manual_scaling, attribute) in ('manual', None):
          setattr(manual_scaling, attribute,
                  getattr(cls._DEFAULT_MANUAL_SCALING, attribute))
  
    def _process_manual_scaling(self, manual_scaling):
      if manual_scaling:
        self._populate_default_manual_scaling(manual_scaling)
      else:
        manual_scaling = self._DEFAULT_MANUAL_SCALING
      self._initial_num_instances = int(manual_scaling.instances)
  
    def __init__(self,
                 module_configuration,
                 host,
                 balanced_port,
                 api_port,
                 auth_domain,
                 runtime_stderr_loglevel,
                 php_executable_path,
                 enable_php_remote_debugging,
                 python_config,
                 cloud_sql_config,
                 default_version_port,
                 port_registry,
                 request_data,
                 dispatcher,
                 max_instances,
                 use_mtime_file_watcher,
                 automatic_restarts,
                 allow_skipped_files):
      """Initializer for ManualScalingModule.
  
      Args:
        module_configuration: An application_configuration.ModuleConfiguration
            instance storing the configuration data for a module.
        host: A string containing the host that any HTTP servers should bind to
            e.g. "localhost".
        balanced_port: An int specifying the port where the balanced module for
            the pool should listen.
        api_port: The port that APIServer listens for RPC requests on.
        auth_domain: A string containing the auth domain to set in the environment
            variables.
        runtime_stderr_loglevel: An int reprenting the minimum logging level at
            which runtime log messages should be written to stderr. See
            devappserver2.py for possible values.
        php_executable_path: A string containing the path to PHP execution e.g.
            "/usr/bin/php-cgi".
        enable_php_remote_debugging: A boolean indicating whether the PHP
            interpreter should be started with XDebug remote debugging enabled.
        python_config: A runtime_config_pb2.PythonConfig instance containing
            Python runtime-specific configuration. If None then defaults are
            used.
        cloud_sql_config: A runtime_config_pb2.CloudSQL instance containing the
            required configuration for local Google Cloud SQL development. If None
            then Cloud SQL will not be available.
        default_version_port: An int containing the port of the default version.
        port_registry: A dispatcher.PortRegistry used to provide the Dispatcher
            with a mapping of port to Module and Instance.
        request_data: A wsgi_request_info.WSGIRequestInfo that will be provided
            with request information for use by API stubs.
        dispatcher: A Dispatcher instance that can be used to make HTTP requests.
        max_instances: The maximum number of instances to create for this module.
            If None then there is no limit on the number of created instances.
        use_mtime_file_watcher: A bool containing whether to use mtime polling to
            monitor file changes even if other options are available on the
            current platform.
        automatic_restarts: If True then instances will be restarted when a
            file or configuration change that effects them is detected.
        allow_skipped_files: If True then all files in the application's directory
            are readable, even if they appear in a static handler or "skip_files"
            directive.
      """
      super(ManualScalingModule, self).__init__(module_configuration,
                                                host,
                                                balanced_port,
                                                api_port,
                                                auth_domain,
                                                runtime_stderr_loglevel,
                                                php_executable_path,
                                                enable_php_remote_debugging,
                                                python_config,
                                                cloud_sql_config,
                                                default_version_port,
                                                port_registry,
                                                request_data,
                                                dispatcher,
                                                max_instances,
                                                use_mtime_file_watcher,
                                                automatic_restarts,
                                                allow_skipped_files)
  
      self._process_manual_scaling(module_configuration.manual_scaling)
  
      self._instances = []  # Protected by self._condition.
      self._wsgi_servers = []  # Protected by self._condition.
      # Whether the module has been stopped. Protected by self._condition.
      self._suspended = False
  
      self._condition = threading.Condition()  # Protects instance state.
  
      # Serializes operations that modify the serving state of or number of
      # instances.
      self._instances_change_lock = threading.RLock()
  
      self._change_watcher_thread = threading.Thread(
          target=self._loop_watching_for_changes)
  
    def start(self):
      """Start background management of the Module."""
      self._balanced_module.start()
      self._port_registry.add(self.balanced_port, self, None)
      if self._watcher:
        self._watcher.start()
      self._change_watcher_thread.start()
      with self._instances_change_lock:
        if self._max_instances is not None:
          initial_num_instances = min(self._max_instances,
                                      self._initial_num_instances)
        else:
          initial_num_instances = self._initial_num_instances
        for _ in xrange(initial_num_instances):
          self._add_instance()
  
    def quit(self):
      """Stops the Module."""
      self._quit_event.set()
      self._change_watcher_thread.join()
      # The instance adjustment thread depends on the balanced module and the
      # watcher so wait for it exit before quitting them.
      if self._watcher:
        self._watcher.quit()
      self._balanced_module.quit()
      for wsgi_servr in self._wsgi_servers:
        wsgi_servr.quit()
      with self._condition:
        instances = self._instances
        self._instances = []
        self._condition.notify_all()
      for inst in instances:
        inst.quit(force=True)
  
    def get_instance_port(self, instance_id):
      """Returns the port of the HTTP server for an instance."""
      try:
        instance_id = int(instance_id)
      except ValueError:
        raise request_info.InvalidInstanceIdError()
      with self._condition:
        if 0 <= instance_id < len(self._instances):
          wsgi_servr = self._wsgi_servers[instance_id]
        else:
          raise request_info.InvalidInstanceIdError()
      return wsgi_servr.port
  
    @property
    def instances(self):
      """A set of all the instances currently in the Module."""
      with self._condition:
        return set(self._instances)
  
    def _handle_instance_request(self,
                                 environ,
                                 start_response,
                                 url_map,
                                 match,
                                 request_id,
                                 inst,
                                 request_type):
      """Handles a request routed a particular Instance.
  
      Args:
        environ: An environ dict for the request as defined in PEP-333.
        start_response: A function with semantics defined in PEP-333.
        url_map: An appinfo.URLMap instance containing the configuration for the
            handler that matched.
        match: A re.MatchObject containing the result of the matched URL pattern.
        request_id: A unique string id associated with the request.
        inst: The instance.Instance to send the request to.
        request_type: The type of the request. See instance.*_REQUEST module
            constants.
  
      Returns:
        An iterable over strings containing the body of the HTTP response.
      """
      start_time = time.time()
      timeout_time = start_time + self._MAX_REQUEST_WAIT_TIME
      try:
        while time.time() < timeout_time:
          logging.debug('Dispatching request to %s after %0.4fs pending',
                        inst, time.time() - start_time)
          try:
            return inst.handle(environ, start_response, url_map, match,
                               request_id, request_type)
          except instance.CannotAcceptRequests:
            pass
          inst.wait(timeout_time)
          if inst.has_quit:
            return self._error_response(environ, start_response, 503)
        else:
          return self._error_response(environ, start_response, 503)
      finally:
        with self._condition:
          self._condition.notify()
  
    def _handle_script_request(self,
                               environ,
                               start_response,
                               url_map,
                               match,
                               request_id,
                               inst=None,
                               request_type=instance.NORMAL_REQUEST):
      """Handles a HTTP request that has matched a script handler.
  
      Args:
        environ: An environ dict for the request as defined in PEP-333.
        start_response: A function with semantics defined in PEP-333.
        url_map: An appinfo.URLMap instance containing the configuration for the
            handler that matched.
        match: A re.MatchObject containing the result of the matched URL pattern.
        request_id: A unique string id associated with the request.
        inst: The instance.Instance to send the request to. If None then an
            appropriate instance.Instance will be chosen.
        request_type: The type of the request. See instance.*_REQUEST module
            constants.
  
      Returns:
        An iterable over strings containing the body of the HTTP response.
      """
      if ((request_type in (instance.NORMAL_REQUEST, instance.READY_REQUEST) and
           self._suspended) or self._quit_event.is_set()):
        return self._error_response(environ, start_response, 404)
      if self._module_configuration.is_backend:
        environ['BACKEND_ID'] = self._module_configuration.module_name
      else:
        environ['BACKEND_ID'] = (
            self._module_configuration.version_id.split('.', 1)[0])
      if inst is not None:
        return self._handle_instance_request(
            environ, start_response, url_map, match, request_id, inst,
            request_type)
  
      start_time = time.time()
      timeout_time = start_time + self._MAX_REQUEST_WAIT_TIME
      while time.time() < timeout_time:
        if ((request_type in (instance.NORMAL_REQUEST, instance.READY_REQUEST) and
             self._suspended) or self._quit_event.is_set()):
          return self._error_response(environ, start_response, 404)
        inst = self._choose_instance(timeout_time)
        if inst:
          try:
            logging.debug('Dispatching request to %s after %0.4fs pending',
                          inst, time.time() - start_time)
            return inst.handle(environ, start_response, url_map, match,
                               request_id, request_type)
          except instance.CannotAcceptRequests:
            continue
          finally:
            with self._condition:
              self._condition.notify()
      else:
        return self._error_response(environ, start_response, 503)
  
    def _add_instance(self):
      """Creates and adds a new instance.Instance to the Module.
  
      This must be called with _instances_change_lock held.
      """
      instance_id = self.get_num_instances()
      assert self._max_instances is None or instance_id < self._max_instances
      inst = self._instance_factory.new_instance(instance_id,
                                                 expect_ready_request=True)
      wsgi_servr = wsgi_server.WsgiServer(
          (self._host, 0), functools.partial(self._handle_request, inst=inst))
      wsgi_servr.start()
      self._port_registry.add(wsgi_servr.port, self, inst)
      with self._condition:
        if self._quit_event.is_set():
          return
        self._wsgi_servers.append(wsgi_servr)
        self._instances.append(inst)
        suspended = self._suspended
      if not suspended:
        self._async_start_instance(wsgi_servr, inst)
  
    def _async_start_instance(self, wsgi_servr, inst):
      _THREAD_POOL.submit(self._start_instance, wsgi_servr, inst)
  
    def _start_instance(self, wsgi_servr, inst):
      if inst.start():
        logging.debug('Started instance: %s at http://%s:%s', inst, self.host,
                      wsgi_servr.port)
        try:
          environ = self.build_request_environ(
              'GET', '/_ah/start', [], '', '0.1.0.3', wsgi_servr.port,
              fake_login=True)
          self._handle_request(environ,
                               start_response_utils.null_start_response,
                               inst=inst,
                               request_type=instance.READY_REQUEST)
          logging.debug('Sent start request: %s', inst)
          with self._condition:
            self._condition.notify(self.max_instance_concurrent_requests)
        except:
          logging.exception('Internal error while handling start request.')
  
    def _choose_instance(self, timeout_time):
      """Returns an Instance to handle a request or None if all are busy."""
      with self._condition:
        while time.time() < timeout_time:
          for inst in self._instances:
            if inst.can_accept_requests:
              return inst
          self._condition.wait(timeout_time - time.time())
        return None
  
    def _handle_changes(self):
      """Handle file or configuration changes."""
      # Always check for config and file changes because checking also clears
      # pending changes.
      config_changes = self._module_configuration.check_for_updates()
      has_file_changes = self._watcher.has_changes()
  
      if application_configuration.HANDLERS_CHANGED in config_changes:
        handlers = self._create_url_handlers()
        with self._handler_lock:
          self._handlers = handlers
  
      if has_file_changes:
        self._instance_factory.files_changed()
  
      if config_changes & _RESTART_INSTANCES_CONFIG_CHANGES:
        self._instance_factory.configuration_changed(config_changes)
  
      if config_changes & _RESTART_INSTANCES_CONFIG_CHANGES or has_file_changes:
        with self._instances_change_lock:
          if not self._suspended:
            self.restart()
  
    def _loop_watching_for_changes(self):
      """Loops until the InstancePool is done watching for file changes."""
      while not self._quit_event.is_set():
        if self.ready:
          if self._automatic_restarts:
            self._handle_changes()
        self._quit_event.wait(timeout=1)
  
    def get_num_instances(self):
      with self._instances_change_lock:
        with self._condition:
          return len(self._instances)
  
    def set_num_instances(self, instances):
      if self._max_instances is not None:
        instances = min(instances, self._max_instances)
  
      with self._instances_change_lock:
        with self._condition:
          running_instances = self.get_num_instances()
          if running_instances > instances:
            wsgi_servers_to_quit = self._wsgi_servers[instances:]
            del self._wsgi_servers[instances:]
            instances_to_quit = self._instances[instances:]
            del self._instances[instances:]
        if running_instances < instances:
          for _ in xrange(instances - running_instances):
            self._add_instance()
      if running_instances > instances:
        for inst, wsgi_servr in zip(instances_to_quit, wsgi_servers_to_quit):
          self._async_quit_instance(inst, wsgi_servr)
  
    def _async_quit_instance(self, inst, wsgi_servr):
      _THREAD_POOL.submit(self._quit_instance, inst, wsgi_servr)
  
    def _quit_instance(self, inst, wsgi_servr):
      port = wsgi_servr.port
      wsgi_servr.quit()
      inst.quit(expect_shutdown=True)
      self._shutdown_instance(inst, port)
  
    def suspend(self):
      """Suspends serving for this module, quitting all running instances."""
      with self._instances_change_lock:
        if self._suspended:
          raise request_info.ModuleAlreadyStoppedError()
        self._suspended = True
        with self._condition:
          instances_to_stop = zip(self._instances, self._wsgi_servers)
          for wsgi_servr in self._wsgi_servers:
            wsgi_servr.set_error(404)
      for inst, wsgi_servr in instances_to_stop:
        self._async_suspend_instance(inst, wsgi_servr.port)
  
    def _async_suspend_instance(self, inst, port):
      _THREAD_POOL.submit(self._suspend_instance, inst, port)
  
    def _suspend_instance(self, inst, port):
      inst.quit(expect_shutdown=True)
      self._shutdown_instance(inst, port)
  
    def resume(self):
      """Resumes serving for this module."""
      with self._instances_change_lock:
        if not self._suspended:
          raise request_info.ModuleAlreadyStartedError()
        self._suspended = False
        with self._condition:
          if self._quit_event.is_set():
            return
          wsgi_servers = self._wsgi_servers
        instances_to_start = []
        for instance_id, wsgi_servr in enumerate(wsgi_servers):
          inst = self._instance_factory.new_instance(instance_id,
                                                     expect_ready_request=True)
          wsgi_servr.set_app(functools.partial(self._handle_request, inst=inst))
          self._port_registry.add(wsgi_servr.port, self, inst)
          with self._condition:
            if self._quit_event.is_set():
              return
            self._instances[instance_id] = inst
  
          instances_to_start.append((wsgi_servr, inst))
      for wsgi_servr, inst in instances_to_start:
        self._async_start_instance(wsgi_servr, inst)
  
    def restart(self):
      """Restarts the the module, replacing all running instances."""
      with self._instances_change_lock:
        with self._condition:
          if self._quit_event.is_set():
            return
          instances_to_stop = self._instances[:]
          wsgi_servers = self._wsgi_servers[:]
        instances_to_start = []
        for instance_id, wsgi_servr in enumerate(wsgi_servers):
          inst = self._instance_factory.new_instance(instance_id,
                                                     expect_ready_request=True)
          wsgi_servr.set_app(functools.partial(self._handle_request, inst=inst))
          self._port_registry.add(wsgi_servr.port, self, inst)
          instances_to_start.append(inst)
        with self._condition:
          if self._quit_event.is_set():
            return
          self._instances[:] = instances_to_start
      for inst, wsgi_servr in zip(instances_to_stop, wsgi_servers):
        self._async_suspend_instance(inst, wsgi_servr.port)
      for wsgi_servr, inst in zip(wsgi_servers, instances_to_start):
        self._async_start_instance(wsgi_servr, inst)
  
    def get_instance(self, instance_id):
      """Returns the instance with the provided instance ID."""
      try:
        with self._condition:
          return self._instances[int(instance_id)]
      except (ValueError, IndexError):
        raise request_info.InvalidInstanceIdError()
  
    def __call__(self, environ, start_response, inst=None):
      return self._handle_request(environ, start_response, inst)
  
    @property
    def supports_individually_addressable_instances(self):
      return True
  
  
  class BasicScalingModule(Module):
    """A pool of instances that is basic-scaled."""
  
    _DEFAULT_BASIC_SCALING = appinfo.BasicScaling(max_instances='1',
                                                  idle_timeout='15m')
    _MAX_REQUEST_WAIT_TIME = 10
  
    @staticmethod
    def _parse_idle_timeout(timing):
      """Parse a idle timeout string into an int of the value in seconds.
  
      Args:
        timing: A str of the form 1m or 10s.
  
      Returns:
        An int representation of the value in seconds.
      """
      if timing.endswith('m'):
        return int(timing[:-1]) * 60
      else:
        return int(timing[:-1])
  
    @classmethod
    def _populate_default_basic_scaling(cls, basic_scaling):
      for attribute in basic_scaling.ATTRIBUTES:
        if getattr(basic_scaling, attribute) in ('basic', None):
          setattr(basic_scaling, attribute,
                  getattr(cls._DEFAULT_BASIC_SCALING, attribute))
  
    def _process_basic_scaling(self, basic_scaling):
      if basic_scaling:
        self._populate_default_basic_scaling(basic_scaling)
      else:
        basic_scaling = self._DEFAULT_BASIC_SCALING
      if self._max_instances is not None:
        self._max_instances = min(self._max_instances,
                                  int(basic_scaling.max_instances))
      else:
        self._max_instances = int(basic_scaling.max_instances)
      self._instance_idle_timeout = self._parse_idle_timeout(
          basic_scaling.idle_timeout)
  
    def __init__(self,
                 module_configuration,
                 host,
                 balanced_port,
                 api_port,
                 auth_domain,
                 runtime_stderr_loglevel,
                 php_executable_path,
                 enable_php_remote_debugging,
                 python_config,
                 cloud_sql_config,
                 default_version_port,
                 port_registry,
                 request_data,
                 dispatcher,
                 max_instances,
                 use_mtime_file_watcher,
                 automatic_restarts,
                 allow_skipped_files):
      """Initializer for BasicScalingModule.
  
      Args:
        module_configuration: An application_configuration.ModuleConfiguration
            instance storing the configuration data for a module.
        host: A string containing the host that any HTTP servers should bind to
            e.g. "localhost".
        balanced_port: An int specifying the port where the balanced module for
            the pool should listen.
        api_port: The port that APIServer listens for RPC requests on.
        auth_domain: A string containing the auth domain to set in the environment
            variables.
        runtime_stderr_loglevel: An int reprenting the minimum logging level at
            which runtime log messages should be written to stderr. See
            devappserver2.py for possible values.
        php_executable_path: A string containing the path to PHP execution e.g.
            "/usr/bin/php-cgi".
        enable_php_remote_debugging: A boolean indicating whether the PHP
            interpreter should be started with XDebug remote debugging enabled.
        python_config: A runtime_config_pb2.PythonConfig instance containing
            Python runtime-specific configuration. If None then defaults are
            used.
        cloud_sql_config: A runtime_config_pb2.CloudSQL instance containing the
            required configuration for local Google Cloud SQL development. If None
            then Cloud SQL will not be available.
        default_version_port: An int containing the port of the default version.
        port_registry: A dispatcher.PortRegistry used to provide the Dispatcher
            with a mapping of port to Module and Instance.
        request_data: A wsgi_request_info.WSGIRequestInfo that will be provided
            with request information for use by API stubs.
        dispatcher: A Dispatcher instance that can be used to make HTTP requests.
        max_instances: The maximum number of instances to create for this module.
            If None then there is no limit on the number of created instances.
        use_mtime_file_watcher: A bool containing whether to use mtime polling to
            monitor file changes even if other options are available on the
            current platform.
        automatic_restarts: If True then instances will be restarted when a
            file or configuration change that effects them is detected.
        allow_skipped_files: If True then all files in the application's directory
            are readable, even if they appear in a static handler or "skip_files"
            directive.
      """
      super(BasicScalingModule, self).__init__(module_configuration,
                                               host,
                                               balanced_port,
                                               api_port,
                                               auth_domain,
                                               runtime_stderr_loglevel,
                                               php_executable_path,
                                               enable_php_remote_debugging,
                                               python_config,
                                               cloud_sql_config,
                                               default_version_port,
                                               port_registry,
                                               request_data,
                                               dispatcher,
                                               max_instances,
                                               use_mtime_file_watcher,
                                               automatic_restarts,
                                               allow_skipped_files)
      self._process_basic_scaling(module_configuration.basic_scaling)
  
      self._instances = []  # Protected by self._condition.
      self._wsgi_servers = []  # Protected by self._condition.
      # A list of booleans signifying whether the corresponding instance in
      # self._instances has been or is being started.
      self._instance_running = []  # Protected by self._condition.
  
      for instance_id in xrange(self._max_instances):
        inst = self._instance_factory.new_instance(instance_id,
                                                   expect_ready_request=True)
        self._instances.append(inst)
        self._wsgi_servers.append(wsgi_server.WsgiServer(
            (self._host, 0), functools.partial(self._handle_request, inst=inst)))
        self._instance_running.append(False)
  
      self._condition = threading.Condition()  # Protects instance state.
  
      self._change_watcher_thread = threading.Thread(
          target=self._loop_watching_for_changes_and_idle_instances)
  
    def start(self):
      """Start background management of the Module."""
      self._balanced_module.start()
      self._port_registry.add(self.balanced_port, self, None)
      if self._watcher:
        self._watcher.start()
      self._change_watcher_thread.start()
      for wsgi_servr, inst in zip(self._wsgi_servers, self._instances):
        wsgi_servr.start()
        self._port_registry.add(wsgi_servr.port, self, inst)
  
    def quit(self):
      """Stops the Module."""
      self._quit_event.set()
      self._change_watcher_thread.join()
      # The instance adjustment thread depends on the balanced module and the
      # watcher so wait for it exit before quitting them.
      if self._watcher:
        self._watcher.quit()
      self._balanced_module.quit()
      for wsgi_servr in self._wsgi_servers:
        wsgi_servr.quit()
      with self._condition:
        instances = self._instances
        self._instances = []
        self._condition.notify_all()
      for inst in instances:
        inst.quit(force=True)
  
    def get_instance_port(self, instance_id):
      """Returns the port of the HTTP server for an instance."""
      try:
        instance_id = int(instance_id)
      except ValueError:
        raise request_info.InvalidInstanceIdError()
      with self._condition:
        if 0 <= instance_id < len(self._instances):
          wsgi_servr = self._wsgi_servers[instance_id]
        else:
          raise request_info.InvalidInstanceIdError()
      return wsgi_servr.port
  
    @property
    def instances(self):
      """A set of all the instances currently in the Module."""
      with self._condition:
        return set(self._instances)
  
    def _handle_instance_request(self,
                                 environ,
                                 start_response,
                                 url_map,
                                 match,
                                 request_id,
                                 inst,
                                 request_type):
      """Handles a request routed a particular Instance.
  
      Args:
        environ: An environ dict for the request as defined in PEP-333.
        start_response: A function with semantics defined in PEP-333.
        url_map: An appinfo.URLMap instance containing the configuration for the
            handler that matched.
        match: A re.MatchObject containing the result of the matched URL pattern.
        request_id: A unique string id associated with the request.
        inst: The instance.Instance to send the request to.
        request_type: The type of the request. See instance.*_REQUEST module
            constants.
  
      Returns:
        An iterable over strings containing the body of the HTTP response.
      """
      instance_id = inst.instance_id
      start_time = time.time()
      timeout_time = start_time + self._MAX_REQUEST_WAIT_TIME
      try:
        while time.time() < timeout_time:
          logging.debug('Dispatching request to %s after %0.4fs pending',
                        inst, time.time() - start_time)
          try:
            return inst.handle(environ, start_response, url_map, match,
                               request_id, request_type)
          except instance.CannotAcceptRequests:
            pass
          if inst.has_quit:
            return self._error_response(environ, start_response, 503)
          with self._condition:
            if self._instance_running[instance_id]:
              should_start = False
            else:
              self._instance_running[instance_id] = True
              should_start = True
          if should_start:
            self._start_instance(instance_id)
          else:
            inst.wait(timeout_time)
        else:
          return self._error_response(environ, start_response, 503)
      finally:
        with self._condition:
          self._condition.notify()
  
    def _handle_script_request(self,
                               environ,
                               start_response,
                               url_map,
                               match,
                               request_id,
                               inst=None,
                               request_type=instance.NORMAL_REQUEST):
      """Handles a HTTP request that has matched a script handler.
  
      Args:
        environ: An environ dict for the request as defined in PEP-333.
        start_response: A function with semantics defined in PEP-333.
        url_map: An appinfo.URLMap instance containing the configuration for the
            handler that matched.
        match: A re.MatchObject containing the result of the matched URL pattern.
        request_id: A unique string id associated with the request.
        inst: The instance.Instance to send the request to. If None then an
            appropriate instance.Instance will be chosen.
        request_type: The type of the request. See instance.*_REQUEST module
            constants.
  
      Returns:
        An iterable over strings containing the body of the HTTP response.
      """
      if self._quit_event.is_set():
        return self._error_response(environ, start_response, 404)
      if self._module_configuration.is_backend:
        environ['BACKEND_ID'] = self._module_configuration.module_name
      else:
        environ['BACKEND_ID'] = (
            self._module_configuration.version_id.split('.', 1)[0])
      if inst is not None:
        return self._handle_instance_request(
            environ, start_response, url_map, match, request_id, inst,
            request_type)
  
      start_time = time.time()
      timeout_time = start_time + self._MAX_REQUEST_WAIT_TIME
      while time.time() < timeout_time:
        if self._quit_event.is_set():
          return self._error_response(environ, start_response, 404)
        inst = self._choose_instance(timeout_time)
        if inst:
          try:
            logging.debug('Dispatching request to %s after %0.4fs pending',
                          inst, time.time() - start_time)
            return inst.handle(environ, start_response, url_map, match,
                               request_id, request_type)
          except instance.CannotAcceptRequests:
            continue
          finally:
            with self._condition:
              self._condition.notify()
      else:
        return self._error_response(environ, start_response, 503)
  
    def _start_any_instance(self):
      """Choose an inactive instance and start it asynchronously.
  
      Returns:
        An instance.Instance that will be started asynchronously or None if all
        instances are already running.
      """
      with self._condition:
        for instance_id, running in enumerate(self._instance_running):
          if not running:
            self._instance_running[instance_id] = True
            inst = self._instances[instance_id]
            break
        else:
          return None
      self._async_start_instance(instance_id)
      return inst
  
    def _async_start_instance(self, instance_id):
      _THREAD_POOL.submit(self._start_instance, instance_id)
  
    def _start_instance(self, instance_id):
      with self._condition:
        if self._quit_event.is_set():
          return
        wsgi_servr = self._wsgi_servers[instance_id]
        inst = self._instances[instance_id]
      if inst.start():
        logging.debug('Started instance: %s at http://%s:%s', inst, self.host,
                      wsgi_servr.port)
        try:
          environ = self.build_request_environ(
              'GET', '/_ah/start', [], '', '0.1.0.3', wsgi_servr.port,
              fake_login=True)
          self._handle_request(environ,
                               start_response_utils.null_start_response,
                               inst=inst,
                               request_type=instance.READY_REQUEST)
          logging.debug('Sent start request: %s', inst)
          with self._condition:
            self._condition.notify(self.max_instance_concurrent_requests)
        except:
          logging.exception('Internal error while handling start request.')
  
    def _choose_instance(self, timeout_time):
      """Returns an Instance to handle a request or None if all are busy."""
      with self._condition:
        while time.time() < timeout_time and not self._quit_event.is_set():
          for inst in self._instances:
            if inst.can_accept_requests:
              return inst
          else:
            inst = self._start_any_instance()
            if inst:
              break
            self._condition.wait(timeout_time - time.time())
        else:
          return None
      if inst:
        inst.wait(timeout_time)
      return inst
  
    def _handle_changes(self):
      """Handle file or configuration changes."""
      # Always check for config and file changes because checking also clears
      # pending changes.
      config_changes = self._module_configuration.check_for_updates()
      has_file_changes = self._watcher.has_changes()
  
      if application_configuration.HANDLERS_CHANGED in config_changes:
        handlers = self._create_url_handlers()
        with self._handler_lock:
          self._handlers = handlers
  
      if has_file_changes:
        self._instance_factory.files_changed()
  
      if config_changes & _RESTART_INSTANCES_CONFIG_CHANGES:
        self._instance_factory.configuration_changed(config_changes)
  
      if config_changes & _RESTART_INSTANCES_CONFIG_CHANGES or has_file_changes:
        self.restart()
  
    def _loop_watching_for_changes_and_idle_instances(self):
      """Loops until the InstancePool is done watching for file changes."""
      while not self._quit_event.is_set():
        if self.ready:
          self._shutdown_idle_instances()
          if self._automatic_restarts:
            self._handle_changes()
        self._quit_event.wait(timeout=1)
  
    def _shutdown_idle_instances(self):
      instances_to_stop = []
      with self._condition:
        for instance_id, inst in enumerate(self._instances):
          if (self._instance_running[instance_id] and
              inst.idle_seconds > self._instance_idle_timeout):
            instances_to_stop.append((self._instances[instance_id],
                                      self._wsgi_servers[instance_id]))
            self._instance_running[instance_id] = False
            new_instance = self._instance_factory.new_instance(
                instance_id, expect_ready_request=True)
            self._instances[instance_id] = new_instance
            wsgi_servr = self._wsgi_servers[instance_id]
            wsgi_servr.set_app(
                functools.partial(self._handle_request, inst=new_instance))
            self._port_registry.add(wsgi_servr.port, self, new_instance)
      for inst, wsgi_servr in instances_to_stop:
        logging.debug('Shutting down %r', inst)
        self._stop_instance(inst, wsgi_servr)
  
    def _stop_instance(self, inst, wsgi_servr):
      inst.quit(expect_shutdown=True)
      self._async_shutdown_instance(inst, wsgi_servr.port)
  
    def restart(self):
      """Restarts the the module, replacing all running instances."""
      instances_to_stop = []
      instances_to_start = []
      with self._condition:
        if self._quit_event.is_set():
          return
        for instance_id, inst in enumerate(self._instances):
          if self._instance_running[instance_id]:
            instances_to_stop.append((inst, self._wsgi_servers[instance_id]))
            new_instance = self._instance_factory.new_instance(
                instance_id, expect_ready_request=True)
            self._instances[instance_id] = new_instance
            instances_to_start.append(instance_id)
            wsgi_servr = self._wsgi_servers[instance_id]
            wsgi_servr.set_app(
                functools.partial(self._handle_request, inst=new_instance))
            self._port_registry.add(wsgi_servr.port, self, new_instance)
      for instance_id in instances_to_start:
        self._async_start_instance(instance_id)
      for inst, wsgi_servr in instances_to_stop:
        self._stop_instance(inst, wsgi_servr)
  
    def get_instance(self, instance_id):
      """Returns the instance with the provided instance ID."""
      try:
        with self._condition:
          return self._instances[int(instance_id)]
      except (ValueError, IndexError):
        raise request_info.InvalidInstanceIdError()
  
    def __call__(self, environ, start_response, inst=None):
      return self._handle_request(environ, start_response, inst)
  
    @property
    def supports_individually_addressable_instances(self):
      return True
  
  
  class InteractiveCommandModule(Module):
    """A Module that can evaluate user commands.
  
    This module manages a single Instance which is started lazily.
    """
  
    _MAX_REQUEST_WAIT_TIME = 15
  
    def __init__(self,
                 module_configuration,
                 host,
                 balanced_port,
                 api_port,
                 auth_domain,
                 runtime_stderr_loglevel,
                 php_executable_path,
                 enable_php_remote_debugging,
                 python_config,
                 cloud_sql_config,
                 default_version_port,
                 port_registry,
                 request_data,
                 dispatcher,
                 use_mtime_file_watcher,
                 allow_skipped_files):
      """Initializer for InteractiveCommandModule.
  
      Args:
        module_configuration: An application_configuration.ModuleConfiguration
            instance storing the configuration data for this module.
        host: A string containing the host that will be used when constructing
            HTTP headers sent to the Instance executing the interactive command
            e.g. "localhost".
        balanced_port: An int specifying the port that will be used when
            constructing HTTP headers sent to the Instance executing the
            interactive command e.g. "localhost".
        api_port: The port that APIServer listens for RPC requests on.
        auth_domain: A string containing the auth domain to set in the environment
            variables.
        runtime_stderr_loglevel: An int reprenting the minimum logging level at
            which runtime log messages should be written to stderr. See
            devappserver2.py for possible values.
        php_executable_path: A string containing the path to PHP execution e.g.
            "/usr/bin/php-cgi".
        enable_php_remote_debugging: A boolean indicating whether the PHP
            interpreter should be started with XDebug remote debugging enabled.
        python_config: A runtime_config_pb2.PythonConfig instance containing
            Python runtime-specific configuration. If None then defaults are
            used.
        cloud_sql_config: A runtime_config_pb2.CloudSQL instance containing the
            required configuration for local Google Cloud SQL development. If None
            then Cloud SQL will not be available.
        default_version_port: An int containing the port of the default version.
        port_registry: A dispatcher.PortRegistry used to provide the Dispatcher
            with a mapping of port to Module and Instance.
        request_data: A wsgi_request_info.WSGIRequestInfo that will be provided
            with request information for use by API stubs.
        dispatcher: A Dispatcher instance that can be used to make HTTP requests.
        use_mtime_file_watcher: A bool containing whether to use mtime polling to
            monitor file changes even if other options are available on the
            current platform.
        allow_skipped_files: If True then all files in the application's directory
            are readable, even if they appear in a static handler or "skip_files"
            directive.
      """
      super(InteractiveCommandModule, self).__init__(
          module_configuration,
          host,
          balanced_port,
          api_port,
          auth_domain,
          runtime_stderr_loglevel,
          php_executable_path,
          enable_php_remote_debugging,
          python_config,
          cloud_sql_config,
          default_version_port,
          port_registry,
          request_data,
          dispatcher,
          max_instances=1,
          use_mtime_file_watcher=use_mtime_file_watcher,
          automatic_restarts=True,
          allow_skipped_files=allow_skipped_files)
      # Use a single instance so that state is consistent across requests.
      self._inst_lock = threading.Lock()
      self._inst = None
  
    @property
    def balanced_port(self):
      """The port that the balanced HTTP server for the Module is listening on.
  
      The InteractiveCommandModule does not actually listen on this port but it is
      used when constructing the "SERVER_PORT" in the WSGI-environment.
      """
      return self._balanced_port
  
    def quit(self):
      """Stops the InteractiveCommandModule."""
      if self._inst:
        self._inst.quit(force=True)
        self._inst = None
  
    def _handle_script_request(self,
                               environ,
                               start_response,
                               url_map,
                               match,
                               request_id,
                               inst=None,
                               request_type=instance.INTERACTIVE_REQUEST):
      """Handles a interactive request by forwarding it to the managed Instance.
  
      Args:
        environ: An environ dict for the request as defined in PEP-333.
        start_response: A function with semantics defined in PEP-333.
        url_map: An appinfo.URLMap instance containing the configuration for the
            handler that matched.
        match: A re.MatchObject containing the result of the matched URL pattern.
        request_id: A unique string id associated with the request.
        inst: The instance.Instance to send the request to.
        request_type: The type of the request. See instance.*_REQUEST module
            constants. This must be instance.INTERACTIVE_REQUEST.
  
      Returns:
        An iterable over strings containing the body of the HTTP response.
      """
      assert inst is None
      assert request_type == instance.INTERACTIVE_REQUEST
  
      start_time = time.time()
      timeout_time = start_time + self._MAX_REQUEST_WAIT_TIME
  
      while time.time() < timeout_time:
        new_instance = False
        with self._inst_lock:
          if not self._inst:
            self._inst = self._instance_factory.new_instance(
                AutoScalingModule.generate_instance_id(),
                expect_ready_request=False)
            new_instance = True
          inst = self._inst
  
        if new_instance:
          self._inst.start()
  
        try:
          return inst.handle(environ, start_response, url_map, match,
                             request_id, request_type)
        except instance.CannotAcceptRequests:
          inst.wait(timeout_time)
        except Exception:
          # If the instance is restarted while handling a request then the
          # exception raises is unpredictable.
          if inst != self._inst:
            start_response('503 Service Unavailable', [])
            return ['Instance was restarted while executing command']
          logging.exception('Unexpected exception handling command: %r', environ)
          raise
      else:
        start_response('503 Service Unavailable', [])
        return ['The command timed-out while waiting for another one to complete']
  
    def restart(self):
      """Restarts the the module."""
      with self._inst_lock:
        if self._inst:
          self._inst.quit(force=True)
          self._inst = None
  
    def send_interactive_command(self, command):
      """Sends an interactive command to the module.
  
      Args:
        command: The command to send e.g. "print 5+5".
  
      Returns:
        A string representing the result of the command e.g. "10\n".
  
      Raises:
        InteractiveCommandError: if the command failed for any reason.
      """
      start_response = start_response_utils.CapturingStartResponse()
  
      # 192.0.2.0 is an example address defined in RFC 5737.
      environ = self.build_request_environ(
          'POST', '/', [], command, '192.0.2.0', self.balanced_port)
  
      try:
        response = self._handle_request(
            environ,
            start_response,
            request_type=instance.INTERACTIVE_REQUEST)
      except Exception as e:
        raise InteractiveCommandError('Unexpected command failure: ', str(e))
  
      if start_response.status != '200 OK':
        raise InteractiveCommandError(start_response.merged_response(response))
  
      return start_response.merged_response(response)
  
files: AppServer/google/appengine/tools/devappserver2/module.py
files: #!/usr/bin/env python
  #
  # Copyright 2007 Google Inc.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  #
  """A Python devappserver2 runtime."""
  
  
  import base64
  import os
  import struct
  import sys
  import time
  import traceback
  
  import google
  
  from google.appengine.api import rdbms_mysqldb
  from google.appengine.ext.remote_api import remote_api_stub
  from google.appengine.tools.devappserver2 import request_rewriter
  from google.appengine.tools.devappserver2 import runtime_config_pb2
  from google.appengine.tools.devappserver2 import wsgi_server
  from google.appengine.tools.devappserver2.python import sandbox
  
  
  _STARTUP_FAILURE_TEMPLATE = """
  <!DOCTYPE html>
  <html>
  <head>
  <title>Startup Script Failure</title>
  </head>
  
  <body>
  <b>Debugger startup failed: {exception_message}</b>
  <details>
    <summary>Configuration</summary>
    <pre><code>{config}</code></pre>
  </details>
  <details>
    <summary>Traceback</summary>
    <pre><code>{traceback}</code></pre>
  </details>
  </body>
  </html>"""
  
  
  def setup_stubs(config, external_api_port=None):
    """Sets up API stubs using remote API."""
    if external_api_port is None:
      external_api_server = None
    else:
      external_api_server = 'localhost:{}'.format(external_api_port)
  
    remote_api_stub.ConfigureRemoteApi(config.app_id, '/', lambda: ('', ''),
                                       'localhost:%d' % config.api_port,
                                       use_remote_datastore=False,
                                       use_async_rpc=True,
                                       external_api_server=external_api_server)
  
    if config.HasField('cloud_sql_config'):
      # Connect the RDBMS API to MySQL.
      sys.modules['google.appengine.api.rdbms'] = rdbms_mysqldb
      google.appengine.api.rdbms = rdbms_mysqldb
  
      connect_kwargs = dict(host=config.cloud_sql_config.mysql_host,
                            port=config.cloud_sql_config.mysql_port,
                            user=config.cloud_sql_config.mysql_user,
                            passwd=config.cloud_sql_config.mysql_password)
  
      if config.cloud_sql_config.mysql_socket:
        connect_kwargs['unix_socket'] = config.cloud_sql_config.mysql_socket
      elif (os.name == 'posix' and
            config.cloud_sql_config.mysql_host == 'localhost'):
        # From http://dev.mysql.com/doc/refman/5.0/en/connecting.html:
        # "On Unix, MySQL programs treat the host name localhost specially,
        # in a way that is likely different from what you expect compared to
        # other network-based programs. For connections to localhost, MySQL
        # programs attempt to connect to the local server by using a Unix socket
        # file. This occurs even if a --port or -P option is given to specify a
        # port number."
        #
        # This logic is duplicated in rdbms_mysqldb.connect but FindUnixSocket
        # will not worked in devappserver2 when rdbms_mysqldb.connect is called
        # because os.access is replaced in the sandboxed environment.
        #
        # A warning is not logged if FindUnixSocket returns None because it would
        # appear for all users, not just those who call connect.
        connect_kwargs['unix_socket'] = rdbms_mysqldb.FindUnixSocket()
  
      rdbms_mysqldb.SetConnectKwargs(**connect_kwargs)
  
  
  class StartupScriptFailureApplication(object):
    """A PEP-333 application that displays startup script failure information."""
  
    def __init__(self, config, exception_message, formatted_traceback):
      self._config = config
      self._exception_message = exception_message
      self._formatted_traceback = formatted_traceback
  
    def __call__(self, environ, start_response):
      start_response('500 Internal Server Error',
                     [('Content-Type', 'text/html')])
      yield _STARTUP_FAILURE_TEMPLATE.format(
          exception_message=self._exception_message,
          config=str(self._config),
          traceback=self._formatted_traceback)
  
  
  def expand_user(path):
    """Fake implementation of os.path.expanduser(path)."""
    return path
  
  
  def main():
    config = runtime_config_pb2.Config()
    config.ParseFromString(base64.b64decode(sys.stdin.read()))
  
    # AppScale: The external port is packed in the same field as the API port.
    external_api_port = None
    if config.api_port > 65535:
      port_bytes = struct.pack('I', config.api_port)
      config.api_port, external_api_port = struct.unpack('HH', port_bytes)
  
    debugging_app = None
    if config.python_config and config.python_config.startup_script:
      global_vars = {'config': config}
      try:
        execfile(config.python_config.startup_script, global_vars)
      except Exception, e:
        debugging_app = StartupScriptFailureApplication(config,
                                                        str(e),
                                                        traceback.format_exc())
  
    my_ip_address = None
    with open('/etc/appscale/my_private_ip') as file_handle:
      my_ip_address = file_handle.read().strip()
  
    nginx_host = None
    with open('/etc/appscale/login_ip') as file_handle:
      raw_ips = file_handle.read()
      ips = raw_ips.split('\n')
      nginx_host = ips[0]
  
    if debugging_app:
      server = wsgi_server.WsgiServer(
          ('localhost', 0),
          debugging_app)
    else:
      setup_stubs(config, external_api_port)
      sandbox.enable_sandbox(config)
      os.path.expanduser = expand_user
      # This import needs to be after enabling the sandbox so the runtime
      # implementation imports the sandboxed version of the logging module.
      from google.appengine.tools.devappserver2.python import request_handler
  
      os.environ['MY_IP_ADDRESS'] = my_ip_address
      os.environ['NGINX_HOST'] = nginx_host
      server = wsgi_server.WsgiServer(
          ('localhost', 0),
          request_rewriter.runtime_rewriter_middleware(
              request_handler.RequestHandler(config)))
    server.start()
    print server.port
    sys.stdout.close()
    sys.stdout = sys.stderr
    try:
      while True:
        time.sleep(1)
    except KeyboardInterrupt:
      pass
    finally:
      server.quit()
  
  
  if __name__ == '__main__':
    main()
  
files: AppServer/google/appengine/tools/devappserver2/python/runtime.py
files: #!/usr/bin/env python
  #
  # Copyright 2007 Google Inc.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  #
  """A Python devappserver2 runtime."""
  
  
  import base64
  import os
  import sys
  import time
  import traceback
  
  import google
  
  from google.appengine.api import rdbms_mysqldb
  from google.appengine.ext.remote_api import remote_api_stub
  from google.appengine.tools.devappserver2 import request_rewriter
  from google.appengine.tools.devappserver2 import runtime_config_pb2
  from google.appengine.tools.devappserver2 import wsgi_server
  from google.appengine.tools.devappserver2.python import sandbox
  
  
  _STARTUP_FAILURE_TEMPLATE = """
  <!DOCTYPE html>
  <html>
  <head>
  <title>Startup Script Failure</title>
  </head>
  
  <body>
  <b>Debugger startup failed: {exception_message}</b>
  <details>
    <summary>Configuration</summary>
    <pre><code>{config}</code></pre>
  </details>
  <details>
    <summary>Traceback</summary>
    <pre><code>{traceback}</code></pre>
  </details>
  </body>
  </html>"""
  
  
  def setup_stubs(config):
    """Sets up API stubs using remote API."""
    remote_api_stub.ConfigureRemoteApi(config.app_id, '/', lambda: ('', ''),
                                       'localhost:%d' % config.api_port,
                                       use_remote_datastore=False,
                                       use_async_rpc=True)
  
    if config.HasField('cloud_sql_config'):
      # Connect the RDBMS API to MySQL.
      sys.modules['google.appengine.api.rdbms'] = rdbms_mysqldb
      google.appengine.api.rdbms = rdbms_mysqldb
  
      connect_kwargs = dict(host=config.cloud_sql_config.mysql_host,
                            port=config.cloud_sql_config.mysql_port,
                            user=config.cloud_sql_config.mysql_user,
                            passwd=config.cloud_sql_config.mysql_password)
  
      if config.cloud_sql_config.mysql_socket:
        connect_kwargs['unix_socket'] = config.cloud_sql_config.mysql_socket
      elif (os.name == 'posix' and
            config.cloud_sql_config.mysql_host == 'localhost'):
        # From http://dev.mysql.com/doc/refman/5.0/en/connecting.html:
        # "On Unix, MySQL programs treat the host name localhost specially,
        # in a way that is likely different from what you expect compared to
        # other network-based programs. For connections to localhost, MySQL
        # programs attempt to connect to the local server by using a Unix socket
        # file. This occurs even if a --port or -P option is given to specify a
        # port number."
        #
        # This logic is duplicated in rdbms_mysqldb.connect but FindUnixSocket
        # will not worked in devappserver2 when rdbms_mysqldb.connect is called
        # because os.access is replaced in the sandboxed environment.
        #
        # A warning is not logged if FindUnixSocket returns None because it would
        # appear for all users, not just those who call connect.
        connect_kwargs['unix_socket'] = rdbms_mysqldb.FindUnixSocket()
  
      rdbms_mysqldb.SetConnectKwargs(**connect_kwargs)
  
  
  class StartupScriptFailureApplication(object):
    """A PEP-333 application that displays startup script failure information."""
  
    def __init__(self, config, exception_message, formatted_traceback):
      self._config = config
      self._exception_message = exception_message
      self._formatted_traceback = formatted_traceback
  
    def __call__(self, environ, start_response):
      start_response('500 Internal Server Error',
                     [('Content-Type', 'text/html')])
      yield _STARTUP_FAILURE_TEMPLATE.format(
          exception_message=self._exception_message,
          config=str(self._config),
          traceback=self._formatted_traceback)
  
  
  def expand_user(path):
    """Fake implementation of os.path.expanduser(path)."""
    return path
  
  
  def main():
    config = runtime_config_pb2.Config()
    config.ParseFromString(base64.b64decode(sys.stdin.read()))
    debugging_app = None
    if config.python_config and config.python_config.startup_script:
      global_vars = {'config': config}
      try:
        execfile(config.python_config.startup_script, global_vars)
      except Exception, e:
        debugging_app = StartupScriptFailureApplication(config,
                                                        str(e),
                                                        traceback.format_exc())
  
    my_ip_address = None
    with open('/etc/appscale/my_private_ip') as file_handle:
      my_ip_address = file_handle.read().strip()
  
    nginx_host = None
    with open('/etc/appscale/login_ip') as file_handle:
      raw_ips = file_handle.read()
      ips = raw_ips.split('\n')
      nginx_host = ips[0]
  
    if debugging_app:
      server = wsgi_server.WsgiServer(
          ('localhost', 0),
          debugging_app)
    else:
      setup_stubs(config)
      sandbox.enable_sandbox(config)
      os.path.expanduser = expand_user
      # This import needs to be after enabling the sandbox so the runtime
      # implementation imports the sandboxed version of the logging module.
      from google.appengine.tools.devappserver2.python import request_handler
  
      os.environ['MY_IP_ADDRESS'] = my_ip_address
      os.environ['NGINX_HOST'] = nginx_host
      server = wsgi_server.WsgiServer(
          ('localhost', 0),
          request_rewriter.runtime_rewriter_middleware(
              request_handler.RequestHandler(config)))
    server.start()
    print server.port
    sys.stdout.close()
    sys.stdout = sys.stderr
    try:
      while True:
        time.sleep(1)
    except KeyboardInterrupt:
      pass
    finally:
      server.quit()
  
  
  if __name__ == '__main__':
    main()
  
files: AppServer/google/appengine/tools/devappserver2/python/runtime.py