files: #!/usr/bin/env python
  #
  # Copyright 2007 Google Inc.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  #
  
  
  
  
  
  """Handler for data backup operation.
  
  Generic datastore admin console transfers control to ConfirmBackupHandler
  after selection of entities. The ConfirmBackupHandler confirms with user
  his choice, enters a backup name and transfers control to
  DoBackupHandler. DoBackupHandler starts backup mappers and displays confirmation
  page.
  
  This module also contains actual mapper code for backing data over.
  """
  
  
  
  
  
  
  
  
  from __future__ import with_statement
  
  
  
  import copy
  import cStringIO
  import datetime
  import itertools
  import logging
  import os
  import random
  import re
  import time
  import urllib
  import xml.dom.minidom
  
  from google.appengine._internal import cloudstorage
  
  # AppScale: Use bundled mapreduce library.
  from google.appengine.ext.mapreduce import context
  from google.appengine.ext.mapreduce import datastore_range_iterators as db_iters
  from google.appengine.ext.mapreduce import input_readers
  from google.appengine.ext.mapreduce import json_util
  from google.appengine.ext.mapreduce import operation as op
  from google.appengine.ext.mapreduce import output_writers
  from google.appengine.ext.mapreduce import parameters
  from google.appengine.ext.mapreduce import records
  
  from google.appengine.api import apiproxy_stub_map
  from google.appengine.api import app_identity
  from google.appengine.api import blobstore as blobstore_api
  from google.appengine.api import capabilities
  from google.appengine.api import datastore
  from google.appengine.api import datastore_errors
  from google.appengine.api import datastore_types
  from google.appengine.api import taskqueue
  from google.appengine.api import urlfetch
  from google.appengine.api.taskqueue import taskqueue_service_pb
  from google.appengine.datastore import entity_pb
  from google.appengine.ext import blobstore
  from google.appengine.ext import db
  from google.appengine.ext import deferred
  import webapp2 as webapp
  from google.appengine.runtime import apiproxy_errors
  from google.appengine.ext.datastore_admin import backup_pb2
  from google.appengine.ext.datastore_admin import config
  from google.appengine.ext.datastore_admin import utils
  
  
  XSRF_ACTION = 'backup'
  BUCKET_PATTERN = (r'^([a-zA-Z0-9]+([\-_]+[a-zA-Z0-9]+)*)'
                    r'(\.([a-zA-Z0-9]+([\-_]+[a-zA-Z0-9]+)*))*$')
  MAX_BUCKET_LEN = 222
  MIN_BUCKET_LEN = 3
  MAX_BUCKET_SEGMENT_LEN = 63
  NUM_KINDS_DEFERRED_THRESHOLD = 10
  MAX_BLOBS_PER_DELETE = 500
  TEST_WRITE_FILENAME_PREFIX = 'datastore_backup_write_test'
  MAX_KEYS_LIST_SIZE = 100
  MAX_TEST_FILENAME_TRIES = 10
  
  MEANING_TO_PRIMITIVE_TYPE = {
      entity_pb.Property.GD_WHEN: backup_pb2.EntitySchema.DATE_TIME,
      entity_pb.Property.GD_RATING: backup_pb2.EntitySchema.RATING,
      entity_pb.Property.ATOM_LINK: backup_pb2.EntitySchema.LINK,
      entity_pb.Property.ATOM_CATEGORY: backup_pb2.EntitySchema.CATEGORY,
      entity_pb.Property.GD_PHONENUMBER: backup_pb2.EntitySchema.PHONE_NUMBER,
      entity_pb.Property.GD_POSTALADDRESS: backup_pb2.EntitySchema.POSTAL_ADDRESS,
      entity_pb.Property.GD_EMAIL: backup_pb2.EntitySchema.EMAIL,
      entity_pb.Property.GD_IM: backup_pb2.EntitySchema.IM_HANDLE,
      entity_pb.Property.BLOBKEY: backup_pb2.EntitySchema.BLOB_KEY,
      entity_pb.Property.TEXT: backup_pb2.EntitySchema.TEXT,
      entity_pb.Property.BLOB: backup_pb2.EntitySchema.BLOB,
      entity_pb.Property.BYTESTRING: backup_pb2.EntitySchema.SHORT_BLOB
  }
  
  
  
  
  FILES_API_GS_FILESYSTEM = 'gs'
  FILES_API_BLOBSTORE_FILESYSTEM = 'blobstore'
  FILES_API_BLOBSTORE_PREFIX = '/blobstore/'
  
  BLOBSTORE_BACKUP_DISABLED_FEATURE_NAME = 'DisableBlobstoreBackups'
  
  BLOBSTORE_BACKUP_DISABLED_ERROR_MSG = (
      'Backups to blobstore are disabled, see '
      'https://cloud.google.com/appengine/docs/deprecations/blobstore_backups')
  
  BACKUP_RESTORE_HANDLER = __name__ + '.RestoreEntity.map'
  
  
  def _get_gcs_path_prefix_from_params_dict(params):
    """Returs the gcs_path_prefix from request or mapreduce dict.
  
    Args:
      params: A dict of parameters.
  
    Returns:
      A string containing GCS prefix for the backup. It can be either just the
      bucket or a bucket with a path (as present in the dictionary). Return None
      if 'gs_bucket_name' is not present in the dictionary.
    """
  
  
  
  
  
    return params.get('gs_bucket_name')
  
  
  def _get_basic_mapper_params(handler):
    namespace = handler.request.get('namespace', None)
    if namespace == '*':
      namespace = None
    return {'namespace': namespace}
  
  
  class GCSUtil(object):
    """Wrappers for cloudstorage methods to deal with /gs/ prefix.
  
    Files API requires GCS files to have a /gs/ prefix. cloudstorage treats /gs/
    as a regular bucket name. This class transparently strips the /gs prefix
    before passing the filename to cloudstorage. Also, for listbucket it
    adds the /gs/ back.
  
    backup_handler should not use cloudstorage directly - all access should
    be proxied by GCSUtil.
    """
  
    @classmethod
    def _strip_gs_prefix_if_present(cls, name):
      if name.startswith('/gs/'):
        return name[3:]
      return name
  
    @classmethod
    def add_gs_prefix_if_missing(cls, name):
      if not name.startswith('/gs/'):
        return '/gs%s' % name
      return name
  
    @classmethod
    def open(cls, filename, *args, **kwargs):
      return cloudstorage.open(
          cls._strip_gs_prefix_if_present(filename), *args, **kwargs)
  
    @classmethod
    def listbucket(cls, prefix, *args, **kwargs):
      output = []
      for f in cloudstorage.listbucket(cls._strip_gs_prefix_if_present(prefix),
                                       *args, **kwargs):
        f.filename = cls.add_gs_prefix_if_missing(f.filename)
        output.append(f)
      return output
  
    @classmethod
    def delete(cls, filename, *args, **kwargs):
      return cloudstorage.delete(
          cls._strip_gs_prefix_if_present(filename), *args, **kwargs)
  
  
  def _MaybeStripBlobstorePrefix(blob_key):
    """Strip /blobstore/ so that this reader can accept files API paths."""
    if blob_key.startswith(FILES_API_BLOBSTORE_PREFIX):
      blob_key = blob_key[len(FILES_API_BLOBSTORE_PREFIX):]
    return blob_key
  
  
  class BlobstoreReaderParamsException(Exception):
    pass
  
  
  class BlobstoreRecordsReader(input_readers.InputReader):
    """Reads records from blobstore without files API dependency.
  
    This reader accesses blobstore using (read-only) blobstore API. It only allows
    reading LevelDB records.
  
    On to_json() we record how many bytes we've read and on from_json() we
    seek to the previous location.
  
    Files API prepends blobstore keys with /blobstore/. We need to accept same
    params as files API so we silently strip /blobstore/ from any path.
    """
  
  
    BLOB_KEYS_PARAM = 'files'
  
    def __init__(self, blob_key, start_position):
      self._blob_key = _MaybeStripBlobstorePrefix(blob_key)
      self._records_reader = records.RecordsReader(
          blobstore.BlobReader(self._blob_key))
      self._records_reader.seek(start_position)
  
    @classmethod
    def _get_params(cls, mapper_spec):
  
      return input_readers._get_params(mapper_spec)
  
    @classmethod
    @db.non_transactional(allow_existing=True)
    def validate(cls, mapper_spec):
      params = cls._get_params(mapper_spec)
      if cls.BLOB_KEYS_PARAM not in params or not params[cls.BLOB_KEYS_PARAM]:
        raise BlobstoreReaderParamsException
  
    def __iter__(self):
      for record in self._records_reader:
        yield record
  
    def to_json(self):
      return self._blob_key, self._records_reader.tell()
  
    @classmethod
    def from_json(cls, json):
      return cls(*json)
  
    @classmethod
    def split_input(cls, mapper_spec):
  
      blob_keys = cls._get_params(mapper_spec)[cls.BLOB_KEYS_PARAM]
      return [cls(blob_key, 0) for blob_key in blob_keys]
  
  
  class ConfirmBackupHandler(webapp.RequestHandler):
    """Handler to deal with requests from the admin console to backup data."""
  
    SUFFIX = 'confirm_backup'
  
    @classmethod
    def Render(cls, handler):
      """Rendering method that can be called by main.py.
  
      Args:
        handler: the webapp.RequestHandler invoking the method
      """
      kinds = handler.request.get_all('kind')
      sizes_known, size_total, remainder = utils.ParseKindsAndSizes(kinds)
      notreadonly_warning = capabilities.CapabilitySet(
          'datastore_v3', capabilities=['write']).is_enabled()
      blob_warning = bool(blobstore.BlobInfo.all().count(1))
      template_params = {
          'run_as_a_service': handler.request.get('run_as_a_service'),
          'hide_blobstore': True,
          'form_target': DoBackupHandler.SUFFIX,
          'kind_list': kinds,
          'remainder': remainder,
          'sizes_known': sizes_known,
          'size_total': size_total,
          'queues': None,
          'datastore_admin_home': utils.GenerateHomeUrl(handler.request),
          'namespaces': get_namespaces(handler.request.get('namespace', None)),
          'xsrf_token': utils.CreateXsrfToken(XSRF_ACTION),
          'notreadonly_warning': notreadonly_warning,
          'blob_warning': blob_warning,
          'backup_name': 'datastore_backup_%s' % time.strftime('%Y_%m_%d'),
          'service_accounts': utils.get_service_account_names()
      }
      utils.RenderToResponse(handler, 'confirm_backup.html', template_params)
  
  
  def get_namespaces(selected_namespace):
    namespaces = [('--All--', '*', selected_namespace is None)]
  
  
  
    for ns in datastore.Query('__namespace__', keys_only=True).Run(limit=250):
      ns_name = ns.name() or ''
      namespaces.append((ns_name or '--Default--',
                         ns_name,
                         ns_name == selected_namespace))
    return namespaces
  
  
  class ConfirmDeleteBackupHandler(webapp.RequestHandler):
    """Handler to confirm admin console requests to delete a backup copy."""
  
    SUFFIX = 'confirm_delete
  
  [... the rest is omitted ...]
files: AppServer/google/appengine/ext/datastore_admin/backup_handler.py
files: #!/usr/bin/env python
  #
  # Copyright 2007 Google Inc.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  #
  
  
  
  
  
  """Handler for data backup operation.
  
  Generic datastore admin console transfers control to ConfirmBackupHandler
  after selection of entities. The ConfirmBackupHandler confirms with user
  his choice, enters a backup name and transfers control to
  DoBackupHandler. DoBackupHandler starts backup mappers and displays confirmation
  page.
  
  This module also contains actual mapper code for backing data over.
  """
  
  
  
  
  
  
  
  
  from __future__ import with_statement
  
  
  
  import copy
  import cStringIO
  import datetime
  import itertools
  import json
  import logging
  import os
  import random
  import re
  import time
  import urllib
  import xml.dom.minidom
  
  from google.appengine._internal import cloudstorage
  
  # AppScale: Use bundled mapreduce library.
  from google.appengine.ext.mapreduce import context
  from google.appengine.ext.mapreduce import datastore_range_iterators as db_iters
  from google.appengine.ext.mapreduce import input_readers
  from google.appengine.ext.mapreduce import json_util
  from google.appengine.ext.mapreduce import operation as op
  from google.appengine.ext.mapreduce import output_writers
  from google.appengine.ext.mapreduce import parameters
  from google.appengine.ext.mapreduce import records
  
  from google.appengine.api import apiproxy_stub_map
  from google.appengine.api import app_identity
  from google.appengine.api import blobstore as blobstore_api
  from google.appengine.api import capabilities
  from google.appengine.api import datastore
  from google.appengine.api import datastore_errors
  from google.appengine.api import datastore_types
  from google.appengine.api import taskqueue
  from google.appengine.api import urlfetch
  from google.appengine.api.taskqueue import taskqueue_service_pb
  from google.appengine.datastore import entity_pb
  from google.appengine.ext import blobstore
  from google.appengine.ext import db
  from google.appengine.ext import deferred
  import webapp2 as webapp
  from google.appengine.runtime import apiproxy_errors
  from google.appengine.ext.datastore_admin import backup_pb2
  from google.appengine.ext.datastore_admin import config
  from google.appengine.ext.datastore_admin import utils
  
  
  XSRF_ACTION = 'backup'
  BUCKET_PATTERN = (r'^([a-zA-Z0-9]+([\-_]+[a-zA-Z0-9]+)*)'
                    r'(\.([a-zA-Z0-9]+([\-_]+[a-zA-Z0-9]+)*))*$')
  MAX_BUCKET_LEN = 222
  MIN_BUCKET_LEN = 3
  MAX_BUCKET_SEGMENT_LEN = 63
  NUM_KINDS_DEFERRED_THRESHOLD = 10
  MAX_BLOBS_PER_DELETE = 500
  TEST_WRITE_FILENAME_PREFIX = 'datastore_backup_write_test'
  MAX_KEYS_LIST_SIZE = 100
  MAX_TEST_FILENAME_TRIES = 10
  
  MEANING_TO_PRIMITIVE_TYPE = {
      entity_pb.Property.GD_WHEN: backup_pb2.EntitySchema.DATE_TIME,
      entity_pb.Property.GD_RATING: backup_pb2.EntitySchema.RATING,
      entity_pb.Property.ATOM_LINK: backup_pb2.EntitySchema.LINK,
      entity_pb.Property.ATOM_CATEGORY: backup_pb2.EntitySchema.CATEGORY,
      entity_pb.Property.GD_PHONENUMBER: backup_pb2.EntitySchema.PHONE_NUMBER,
      entity_pb.Property.GD_POSTALADDRESS: backup_pb2.EntitySchema.POSTAL_ADDRESS,
      entity_pb.Property.GD_EMAIL: backup_pb2.EntitySchema.EMAIL,
      entity_pb.Property.GD_IM: backup_pb2.EntitySchema.IM_HANDLE,
      entity_pb.Property.BLOBKEY: backup_pb2.EntitySchema.BLOB_KEY,
      entity_pb.Property.TEXT: backup_pb2.EntitySchema.TEXT,
      entity_pb.Property.BLOB: backup_pb2.EntitySchema.BLOB,
      entity_pb.Property.BYTESTRING: backup_pb2.EntitySchema.SHORT_BLOB
  }
  
  
  
  
  FILES_API_GS_FILESYSTEM = 'gs'
  FILES_API_BLOBSTORE_FILESYSTEM = 'blobstore'
  FILES_API_BLOBSTORE_PREFIX = '/blobstore/'
  
  BLOBSTORE_BACKUP_DISABLED_FEATURE_NAME = 'DisableBlobstoreBackups'
  
  BLOBSTORE_BACKUP_DISABLED_ERROR_MSG = (
      'Backups to blobstore are disabled, see '
      'https://cloud.google.com/appengine/docs/deprecations/blobstore_backups')
  
  BACKUP_RESTORE_HANDLER = __name__ + '.RestoreEntity.map'
  
  
  def get_service_account_names():
    """ AppScale: Fetch list of service accounts from IAM API. """
    project_id = app_identity.get_application_id()
    iam_location = 'https://127.0.0.1:17441'
    url = iam_location + '/v1/projects/{}/serviceAccounts'.format(project_id)
    token = app_identity.get_access_token(
        ['https://www.googleapis.com/auth/cloud-platform'])[0]
    headers = {'Authorization': 'Bearer {}'.format(token)}
    response = urlfetch.fetch(url, headers=headers, validate_certificate=False)
    try:
      accounts = json.loads(response.content)['accounts']
    except (KeyError, ValueError):
      raise ValueError('Invalid list of service accounts: '
                       '{}'.format(response.content))
  
    return tuple(account['email'] for account in accounts)
  
  
  def _get_gcs_path_prefix_from_params_dict(params):
    """Returs the gcs_path_prefix from request or mapreduce dict.
  
    Args:
      params: A dict of parameters.
  
    Returns:
      A string containing GCS prefix for the backup. It can be either just the
      bucket or a bucket with a path (as present in the dictionary). Return None
      if 'gs_bucket_name' is not present in the dictionary.
    """
  
  
  
  
  
    return params.get('gs_bucket_name')
  
  
  def _get_basic_mapper_params(handler):
    namespace = handler.request.get('namespace', None)
    if namespace == '*':
      namespace = None
    return {'namespace': namespace}
  
  
  class GCSUtil(object):
    """Wrappers for cloudstorage methods to deal with /gs/ prefix.
  
    Files API requires GCS files to have a /gs/ prefix. cloudstorage treats /gs/
    as a regular bucket name. This class transparently strips the /gs prefix
    before passing the filename to cloudstorage. Also, for listbucket it
    adds the /gs/ back.
  
    backup_handler should not use cloudstorage directly - all access should
    be proxied by GCSUtil.
    """
  
    @classmethod
    def _strip_gs_prefix_if_present(cls, name):
      if name.startswith('/gs/'):
        return name[3:]
      return name
  
    @classmethod
    def add_gs_prefix_if_missing(cls, name):
      if not name.startswith('/gs/'):
        return '/gs%s' % name
      return name
  
    @classmethod
    def open(cls, filename, *args, **kwargs):
      return cloudstorage.open(
          cls._strip_gs_prefix_if_present(filename), *args, **kwargs)
  
    @classmethod
    def listbucket(cls, prefix, *args, **kwargs):
      output = []
      for f in cloudstorage.listbucket(cls._strip_gs_prefix_if_present(prefix),
                                       *args, **kwargs):
        f.filename = cls.add_gs_prefix_if_missing(f.filename)
        output.append(f)
      return output
  
    @classmethod
    def delete(cls, filename, *args, **kwargs):
      return cloudstorage.delete(
          cls._strip_gs_prefix_if_present(filename), *args, **kwargs)
  
  
  def _MaybeStripBlobstorePrefix(blob_key):
    """Strip /blobstore/ so that this reader can accept files API paths."""
    if blob_key.startswith(FILES_API_BLOBSTORE_PREFIX):
      blob_key = blob_key[len(FILES_API_BLOBSTORE_PREFIX):]
    return blob_key
  
  
  class BlobstoreReaderParamsException(Exception):
    pass
  
  
  class BlobstoreRecordsReader(input_readers.InputReader):
    """Reads records from blobstore without files API dependency.
  
    This reader accesses blobstore using (read-only) blobstore API. It only allows
    reading LevelDB records.
  
    On to_json() we record how many bytes we've read and on from_json() we
    seek to the previous location.
  
    Files API prepends blobstore keys with /blobstore/. We need to accept same
    params as files API so we silently strip /blobstore/ from any path.
    """
  
  
    BLOB_KEYS_PARAM = 'files'
  
    def __init__(self, blob_key, start_position):
      self._blob_key = _MaybeStripBlobstorePrefix(blob_key)
      self._records_reader = records.RecordsReader(
          blobstore.BlobReader(self._blob_key))
      self._records_reader.seek(start_position)
  
    @classmethod
    def _get_params(cls, mapper_spec):
  
      return input_readers._get_params(mapper_spec)
  
    @classmethod
    @db.non_transactional(allow_existing=True)
    def validate(cls, mapper_spec):
      params = cls._get_params(mapper_spec)
      if cls.BLOB_KEYS_PARAM not in params or not params[cls.BLOB_KEYS_PARAM]:
        raise BlobstoreReaderParamsException
  
    def __iter__(self):
      for record in self._records_reader:
        yield record
  
    def to_json(self):
      return self._blob_key, self._records_reader.tell()
  
    @classmethod
    def from_json(cls, json):
      return cls(*json)
  
    @classmethod
    def split_input(cls, mapper_spec):
  
      blob_keys = cls._get_params(mapper_spec)[cls.BLOB_KEYS_PARAM]
      return [cls(blob_key, 0) for blob_key in blob_keys]
  
  
  class ConfirmBackupHandler(webapp.RequestHandler):
    """Handler to deal with requests from the admin console to backup data."""
  
    SUFFIX = 'confirm_backup'
  
    @classmethod
    def Render(cls, handler):
      """Rendering method that can be called by main.py.
  
      Args:
        handler: the webapp.RequestHandler invoking the method
      """
      kinds = handler.request.get_all('kind')
      sizes_known, size_total, remainder = utils.ParseKindsAndSizes(kinds)
      notreadonly_warning = capabilities.CapabilitySet(
          'datastore_v3', capabilities=['write']).is_enabled()
      blob_warning = bool(blobstore.BlobInfo.all().count(1))
      template_params = {
          'run_as_a_service': handler.request.get('run_as_a_service'),
          'hide_blobstore': True,
          'form_target': DoBackupHandler.SUFFIX,
          'kind_list': kinds,
          'remainder': remainder,
          'sizes_known': sizes_known,
          'size_total': size_total,
          'queues': None,
          'datastore_admin_home': utils.GenerateHomeUrl(handler.request),
          'namespaces': get_namespaces(handler.request.get('namespace', None)),
          'xsrf_token': utils.CreateXsrfToken(XSRF_ACTION),
          'notreadonly_warning': notreadonly_warning,
          'blob_warning': blob
  
  [... the rest is omitted ...]
files: AppServer/google/appengine/ext/datastore_admin/backup_handler.py
files: #!/usr/bin/env python
  #
  # Copyright 2007 Google Inc.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  #
  
  
  
  
  """Main module for datastore admin receiver.
  
  To use, add this to app.yaml:
    builtins:
    - datastore_admin: on
  """
  
  
  
  import logging
  import operator
  import os
  
  from google.appengine.api import app_identity
  from google.appengine.api import datastore_errors
  from google.appengine.api import users
  from google.appengine.ext import deferred
  import webapp2 as webapp
  from google.appengine.ext.db import stats
  from google.appengine.ext.webapp import util
  from google.appengine.runtime import apiproxy_errors
  from google.appengine.ext.datastore_admin import backup_handler
  from google.appengine.ext.datastore_admin import config
  from google.appengine.ext.datastore_admin import delete_handler
  from google.appengine.ext.datastore_admin import utils
  
  
  
  
  
  
  ENTITY_ACTIONS = {
      'Delete Entities': delete_handler.ConfirmDeleteHandler.Render,
      'Backup Entities': backup_handler.ConfirmBackupHandler.Render,
  }
  
  
  BACKUP_ACTIONS = {
      'Delete': backup_handler.ConfirmDeleteBackupHandler.Render,
      'Restore': backup_handler.ConfirmRestoreFromBackupHandler.Render,
      'Info': backup_handler.BackupInformationHandler.Render,
  }
  
  PENDING_BACKUP_ACTIONS = {
      'Abort': backup_handler.ConfirmAbortBackupHandler.Render,
      'Info': backup_handler.BackupInformationHandler.Render,
  }
  
  GET_ACTIONS = ENTITY_ACTIONS.copy()
  GET_ACTIONS.update(BACKUP_ACTIONS)
  GET_ACTIONS.update(PENDING_BACKUP_ACTIONS)
  GET_ACTIONS.update({'Import Backup Information':
                      backup_handler.ConfirmBackupImportHandler.Render})
  
  
  def _GetDatastoreStats(kinds_list, use_stats_kinds=False):
    """Retrieves stats for kinds.
  
    Args:
      kinds_list: List of known kinds.
      use_stats_kinds: If stats are available, kinds_list will be ignored and
        all kinds found in stats will be used instead.
  
    Returns:
      timestamp: records time that statistics were last updated.
      kind_dict: dictionary of kind objects with the following members:
      - kind_name: the name of this kind.
      - count: number of known entities of this type.
      - total_bytes_str: total bytes for this kind as a string.
      - average_bytes_str: average bytes per entity as a string.
    """
    global_stat = stats.GlobalStat.all().fetch(1)
    if not global_stat:
      return _KindsListToTuple(kinds_list)
  
    global_ts = global_stat[0].timestamp
  
    kind_stats = stats.KindStat.all().filter('timestamp =', global_ts).fetch(1000)
    if not kind_stats:
      return _KindsListToTuple(kinds_list)
  
    results = {}
    for kind_ent in kind_stats:
  
  
  
      if (not kind_ent.kind_name.startswith('__')
          and (use_stats_kinds or kind_ent.kind_name in kinds_list)
          and kind_ent.count > 0):
        results[kind_ent.kind_name] = _PresentatableKindStats(kind_ent)
  
    utils.CacheStats(results.values())
  
  
  
    for kind_str in kinds_list or []:
      if kind_str not in results:
        results[kind_str] = {'kind_name': kind_str}
  
    return (global_ts,
            sorted(results.values(), key=lambda x: x['kind_name']))
  
  
  def _KindsListToTuple(kinds_list):
    """Build default tuple when no datastore statistics are available. """
    return '', [{'kind_name': kind} for kind in sorted(kinds_list)]
  
  
  def _PresentatableKindStats(kind_ent):
    """Generate dict of presentable values for template."""
    count = kind_ent.count
    entity_bytes = kind_ent.entity_bytes
    total_bytes = kind_ent.bytes
    average_bytes = entity_bytes / count
    return {'kind_name': kind_ent.kind_name,
            'count': utils.FormatThousands(kind_ent.count),
            'entity_bytes_str': utils.GetPrettyBytes(entity_bytes),
            'entity_bytes': entity_bytes,
            'total_bytes_str': utils.GetPrettyBytes(total_bytes),
            'total_bytes': total_bytes,
            'average_bytes_str': utils.GetPrettyBytes(average_bytes),
           }
  
  
  class RouteByActionHandler(webapp.RequestHandler):
    """Route to the appropriate handler based on the action parameter."""
  
    def ListActions(self, error=None):
      """Handler for get requests to datastore_admin/confirm_delete."""
      use_stats_kinds = False
      kinds = []
      more_kinds = False
      try:
        kinds, more_kinds = utils.GetKinds()
        if not kinds:
          use_stats_kinds = True
          logging.warning('Found no kinds. Using datastore stats instead.')
      except datastore_errors.Error, e:
        logging.exception(e)
        use_stats_kinds = True
  
      last_stats_update, kind_stats = _GetDatastoreStats(
          kinds, use_stats_kinds=use_stats_kinds)
  
      template_params = {
          'run_as_a_service': self.request.get('run_as_a_service'),
          'datastore_admin_home': utils.GenerateHomeUrl(None),
          'offer_service': (self.request.get('service') and not
                            self.request.get('run_as_a_service')),
          'kind_stats': kind_stats,
          'more_kinds': more_kinds,
          'last_stats_update': last_stats_update,
          'app_id': self.request.get('app_id'),
          'hosting_app_id': app_identity.get_application_id(),
          'has_namespace': self.request.get('namespace', None) is not None,
          'namespace': self.request.get('namespace'),
          'action_list': sorted(ENTITY_ACTIONS.keys()),
          'backup_action_list': sorted(BACKUP_ACTIONS.keys()),
          'pending_backup_action_list': sorted(PENDING_BACKUP_ACTIONS.keys()),
          'error': error,
          'completed_operations': self.GetOperations(active=False),
          'active_operations': self.GetOperations(active=True),
          'pending_backups': self.GetPendingBackups(),
          'backups': self.GetBackups(),
          'map_reduce_path': config.MAPREDUCE_PATH + '/detail',
          'service_accounts': utils.get_service_account_names()
      }
      utils.RenderToResponse(self, 'list_actions.html', template_params)
  
    def RouteAction(self, action_dict):
      action = self.request.get('action')
      if not action:
        self.ListActions(error=self.request.get('error', None))
      elif action not in action_dict:
        error = '%s is not a valid action.' % action
        self.ListActions(error=error)
      else:
        action_dict[action](self)
  
    def get(self):
      self.RouteAction(GET_ACTIONS)
  
    def post(self):
      self.RouteAction(GET_ACTIONS)
  
    def GetOperations(self, active=False, limit=100):
      """Obtain a list of operation, ordered by last_updated."""
      query = utils.DatastoreAdminOperation.all()
      if active:
        query.filter('status = ', utils.DatastoreAdminOperation.STATUS_ACTIVE)
      else:
        query.filter('status IN ', [
            utils.DatastoreAdminOperation.STATUS_COMPLETED,
            utils.DatastoreAdminOperation.STATUS_FAILED,
            utils.DatastoreAdminOperation.STATUS_ABORTED])
      operations = query.fetch(max(10000, limit) if limit else 1000)
      operations = sorted(operations, key=operator.attrgetter('last_updated'),
                          reverse=True)
      return operations[:limit]
  
    def GetBackups(self, limit=100, deadline=10):
      """Obtain a list of backups.
  
      Args:
        limit: maximum number of backup records to retrieve.
        deadline: maximum number of seconds to spend getting backups.
  
      Returns:
        List of backups, sorted in reverse order by completion time.
      """
      backups = []
      query = backup_handler.BackupInformation.all()
      query.filter('complete_time > ', 0)
      query.order('-complete_time')
      try:
        backups.extend(query.run(deadline=deadline, limit=limit))
      except (datastore_errors.Timeout, apiproxy_errors.DeadlineExceededError):
        logging.warning('Failed to retrieve all backups within deadline.')
      return backups
  
    def GetPendingBackups(self, limit=100):
      """Obtain a list of pending backups."""
      query = backup_handler.BackupInformation.all()
      query.filter('complete_time = ', None)
      backups = query.fetch(max(10000, limit) if limit else 1000)
      backups = sorted(backups, key=operator.attrgetter('start_time'),
                       reverse=True)
      return backups[:limit]
  
  
  class StaticResourceHandler(webapp.RequestHandler):
    """Read static files from disk."""
  
  
  
  
  
  
    _BASE_FILE_PATH = os.path.dirname(__file__)
  
    _RESOURCE_MAP = {
        'static/js/compiled.js': 'text/javascript',
        'static/css/compiled.css': 'text/css',
        'static/img/help.gif': 'image/gif',
        'static/img/tip.png': 'image/png',
        'static/img/icn/icn-warning.gif': 'image/gif',
    }
  
    def get(self):
      relative_path = self.request.path.split(config.BASE_PATH + '/')[1]
      if relative_path not in self._RESOURCE_MAP:
        self.response.set_status(404)
        self.response.out.write('Resource not found.')
        return
  
      path = os.path.join(self._BASE_FILE_PATH, relative_path)
      self.response.headers['Cache-Control'] = 'public; max-age=300'
      self.response.headers['Content-Type'] = self._RESOURCE_MAP[relative_path]
      if relative_path == 'static/css/compiled.css':
  
  
        self.response.out.write(
            open(path).read().replace('url(/img/', 'url(../img/'))
      else:
        self.response.out.write(open(path).read())
  
  
  class LoginRequiredHandler(webapp.RequestHandler):
    """Handle federated login identity selector page."""
  
    def get(self):
      target = self.request.get('continue')
      if not target:
        self.error(400)
        return
  
  
      login_url = users.create_login_url(target)
      self.redirect(login_url)
  
  
  def CreateApplication():
    """Create new WSGIApplication and register all handlers.
  
    Returns:
      an instance of webapp.WSGIApplication with all mapreduce handlers
      registered.
    """
    return webapp.WSGIApplication(
        backup_handler.handlers_list(config.BASE_PATH) +
        [(r'%s/%s' % (config.BASE_PATH,
                      delete_handler.ConfirmDeleteHandler.SUFFIX),
          delete_handler.ConfirmDeleteHandler),
         (r'%s/%s' % (config.BASE_PATH, delete_handler.DoDeleteHandler.SUFFIX),
          delete_handler.DoDeleteHandler),
         (r'%s/%s' % (config.BASE_PATH, utils.MapreduceDoneHandler.SUFFIX),
          utils.MapreduceDoneHandler),
         (config.DEFERRED_PATH, deferred.TaskHandler),
         (r'%s/static.*' % config.BASE_PATH, StaticResourceHandler),
         (r'/_ah/login_required', LoginRequiredHandler),
         (r'.*', RouteByActionHandler)])
  
  
  APP = CreateApplication()
  
  
  def main():
    util.run_wsgi_app(APP)
  
  
  if __name__ == '__main__':
    main()
  
  
  [... the rest is omitted ...]
files: AppServer/google/appengine/ext/datastore_admin/main.py
files: #!/usr/bin/env python
  #
  # Copyright 2007 Google Inc.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  #
  
  
  
  
  """Main module for datastore admin receiver.
  
  To use, add this to app.yaml:
    builtins:
    - datastore_admin: on
  """
  
  
  
  import logging
  import operator
  import os
  
  from google.appengine.api import app_identity
  from google.appengine.api import datastore_errors
  from google.appengine.api import users
  from google.appengine.ext import deferred
  import webapp2 as webapp
  from google.appengine.ext.db import stats
  from google.appengine.ext.webapp import util
  from google.appengine.runtime import apiproxy_errors
  from google.appengine.ext.datastore_admin import backup_handler
  from google.appengine.ext.datastore_admin import config
  from google.appengine.ext.datastore_admin import delete_handler
  from google.appengine.ext.datastore_admin import utils
  
  
  
  
  
  
  ENTITY_ACTIONS = {
      'Delete Entities': delete_handler.ConfirmDeleteHandler.Render,
      'Backup Entities': backup_handler.ConfirmBackupHandler.Render,
  }
  
  
  BACKUP_ACTIONS = {
      'Delete': backup_handler.ConfirmDeleteBackupHandler.Render,
      'Restore': backup_handler.ConfirmRestoreFromBackupHandler.Render,
      'Info': backup_handler.BackupInformationHandler.Render,
  }
  
  PENDING_BACKUP_ACTIONS = {
      'Abort': backup_handler.ConfirmAbortBackupHandler.Render,
      'Info': backup_handler.BackupInformationHandler.Render,
  }
  
  GET_ACTIONS = ENTITY_ACTIONS.copy()
  GET_ACTIONS.update(BACKUP_ACTIONS)
  GET_ACTIONS.update(PENDING_BACKUP_ACTIONS)
  GET_ACTIONS.update({'Import Backup Information':
                      backup_handler.ConfirmBackupImportHandler.Render})
  
  
  def _GetDatastoreStats(kinds_list, use_stats_kinds=False):
    """Retrieves stats for kinds.
  
    Args:
      kinds_list: List of known kinds.
      use_stats_kinds: If stats are available, kinds_list will be ignored and
        all kinds found in stats will be used instead.
  
    Returns:
      timestamp: records time that statistics were last updated.
      kind_dict: dictionary of kind objects with the following members:
      - kind_name: the name of this kind.
      - count: number of known entities of this type.
      - total_bytes_str: total bytes for this kind as a string.
      - average_bytes_str: average bytes per entity as a string.
    """
    global_stat = stats.GlobalStat.all().fetch(1)
    if not global_stat:
      return _KindsListToTuple(kinds_list)
  
    global_ts = global_stat[0].timestamp
  
    kind_stats = stats.KindStat.all().filter('timestamp =', global_ts).fetch(1000)
    if not kind_stats:
      return _KindsListToTuple(kinds_list)
  
    results = {}
    for kind_ent in kind_stats:
  
  
  
      if (not kind_ent.kind_name.startswith('__')
          and (use_stats_kinds or kind_ent.kind_name in kinds_list)
          and kind_ent.count > 0):
        results[kind_ent.kind_name] = _PresentatableKindStats(kind_ent)
  
    utils.CacheStats(results.values())
  
  
  
    for kind_str in kinds_list or []:
      if kind_str not in results:
        results[kind_str] = {'kind_name': kind_str}
  
    return (global_ts,
            sorted(results.values(), key=lambda x: x['kind_name']))
  
  
  def _KindsListToTuple(kinds_list):
    """Build default tuple when no datastore statistics are available. """
    return '', [{'kind_name': kind} for kind in sorted(kinds_list)]
  
  
  def _PresentatableKindStats(kind_ent):
    """Generate dict of presentable values for template."""
    count = kind_ent.count
    entity_bytes = kind_ent.entity_bytes
    total_bytes = kind_ent.bytes
    average_bytes = entity_bytes / count
    return {'kind_name': kind_ent.kind_name,
            'count': utils.FormatThousands(kind_ent.count),
            'entity_bytes_str': utils.GetPrettyBytes(entity_bytes),
            'entity_bytes': entity_bytes,
            'total_bytes_str': utils.GetPrettyBytes(total_bytes),
            'total_bytes': total_bytes,
            'average_bytes_str': utils.GetPrettyBytes(average_bytes),
           }
  
  
  class RouteByActionHandler(webapp.RequestHandler):
    """Route to the appropriate handler based on the action parameter."""
  
    def ListActions(self, error=None):
      """Handler for get requests to datastore_admin/confirm_delete."""
      use_stats_kinds = False
      kinds = []
      more_kinds = False
      try:
        kinds, more_kinds = utils.GetKinds()
        if not kinds:
          use_stats_kinds = True
          logging.warning('Found no kinds. Using datastore stats instead.')
      except datastore_errors.Error, e:
        logging.exception(e)
        use_stats_kinds = True
  
      last_stats_update, kind_stats = _GetDatastoreStats(
          kinds, use_stats_kinds=use_stats_kinds)
  
      template_params = {
          'run_as_a_service': self.request.get('run_as_a_service'),
          'datastore_admin_home': utils.GenerateHomeUrl(None),
          'offer_service': (self.request.get('service') and not
                            self.request.get('run_as_a_service')),
          'kind_stats': kind_stats,
          'more_kinds': more_kinds,
          'last_stats_update': last_stats_update,
          'app_id': self.request.get('app_id'),
          'hosting_app_id': app_identity.get_application_id(),
          'has_namespace': self.request.get('namespace', None) is not None,
          'namespace': self.request.get('namespace'),
          'action_list': sorted(ENTITY_ACTIONS.keys()),
          'backup_action_list': sorted(BACKUP_ACTIONS.keys()),
          'pending_backup_action_list': sorted(PENDING_BACKUP_ACTIONS.keys()),
          'error': error,
          'completed_operations': self.GetOperations(active=False),
          'active_operations': self.GetOperations(active=True),
          'pending_backups': self.GetPendingBackups(),
          'backups': self.GetBackups(),
          'map_reduce_path': config.MAPREDUCE_PATH + '/detail'
      }
      utils.RenderToResponse(self, 'list_actions.html', template_params)
  
    def RouteAction(self, action_dict):
      action = self.request.get('action')
      if not action:
        self.ListActions(error=self.request.get('error', None))
      elif action not in action_dict:
        error = '%s is not a valid action.' % action
        self.ListActions(error=error)
      else:
        action_dict[action](self)
  
    def get(self):
      self.RouteAction(GET_ACTIONS)
  
    def post(self):
      self.RouteAction(GET_ACTIONS)
  
    def GetOperations(self, active=False, limit=100):
      """Obtain a list of operation, ordered by last_updated."""
      query = utils.DatastoreAdminOperation.all()
      if active:
        query.filter('status = ', utils.DatastoreAdminOperation.STATUS_ACTIVE)
      else:
        query.filter('status IN ', [
            utils.DatastoreAdminOperation.STATUS_COMPLETED,
            utils.DatastoreAdminOperation.STATUS_FAILED,
            utils.DatastoreAdminOperation.STATUS_ABORTED])
      operations = query.fetch(max(10000, limit) if limit else 1000)
      operations = sorted(operations, key=operator.attrgetter('last_updated'),
                          reverse=True)
      return operations[:limit]
  
    def GetBackups(self, limit=100, deadline=10):
      """Obtain a list of backups.
  
      Args:
        limit: maximum number of backup records to retrieve.
        deadline: maximum number of seconds to spend getting backups.
  
      Returns:
        List of backups, sorted in reverse order by completion time.
      """
      backups = []
      query = backup_handler.BackupInformation.all()
      query.filter('complete_time > ', 0)
      query.order('-complete_time')
      try:
        backups.extend(query.run(deadline=deadline, limit=limit))
      except (datastore_errors.Timeout, apiproxy_errors.DeadlineExceededError):
        logging.warning('Failed to retrieve all backups within deadline.')
      return backups
  
    def GetPendingBackups(self, limit=100):
      """Obtain a list of pending backups."""
      query = backup_handler.BackupInformation.all()
      query.filter('complete_time = ', None)
      backups = query.fetch(max(10000, limit) if limit else 1000)
      backups = sorted(backups, key=operator.attrgetter('start_time'),
                       reverse=True)
      return backups[:limit]
  
  
  class StaticResourceHandler(webapp.RequestHandler):
    """Read static files from disk."""
  
  
  
  
  
  
    _BASE_FILE_PATH = os.path.dirname(__file__)
  
    _RESOURCE_MAP = {
        'static/js/compiled.js': 'text/javascript',
        'static/css/compiled.css': 'text/css',
        'static/img/help.gif': 'image/gif',
        'static/img/tip.png': 'image/png',
        'static/img/icn/icn-warning.gif': 'image/gif',
    }
  
    def get(self):
      relative_path = self.request.path.split(config.BASE_PATH + '/')[1]
      if relative_path not in self._RESOURCE_MAP:
        self.response.set_status(404)
        self.response.out.write('Resource not found.')
        return
  
      path = os.path.join(self._BASE_FILE_PATH, relative_path)
      self.response.headers['Cache-Control'] = 'public; max-age=300'
      self.response.headers['Content-Type'] = self._RESOURCE_MAP[relative_path]
      if relative_path == 'static/css/compiled.css':
  
  
        self.response.out.write(
            open(path).read().replace('url(/img/', 'url(../img/'))
      else:
        self.response.out.write(open(path).read())
  
  
  class LoginRequiredHandler(webapp.RequestHandler):
    """Handle federated login identity selector page."""
  
    def get(self):
      target = self.request.get('continue')
      if not target:
        self.error(400)
        return
  
  
      login_url = users.create_login_url(target)
      self.redirect(login_url)
  
  
  def CreateApplication():
    """Create new WSGIApplication and register all handlers.
  
    Returns:
      an instance of webapp.WSGIApplication with all mapreduce handlers
      registered.
    """
    return webapp.WSGIApplication(
        backup_handler.handlers_list(config.BASE_PATH) +
        [(r'%s/%s' % (config.BASE_PATH,
                      delete_handler.ConfirmDeleteHandler.SUFFIX),
          delete_handler.ConfirmDeleteHandler),
         (r'%s/%s' % (config.BASE_PATH, delete_handler.DoDeleteHandler.SUFFIX),
          delete_handler.DoDeleteHandler),
         (r'%s/%s' % (config.BASE_PATH, utils.MapreduceDoneHandler.SUFFIX),
          utils.MapreduceDoneHandler),
         (config.DEFERRED_PATH, deferred.TaskHandler),
         (r'%s/static.*' % config.BASE_PATH, StaticResourceHandler),
         (r'/_ah/login_required', LoginRequiredHandler),
         (r'.*', RouteByActionHandler)])
  
  
  APP = CreateApplication()
  
  
  def main():
    util.run_wsgi_app(APP)
  
  
  if __name__ == '__main__':
    main()
  
  
  [... the rest is omitted ...]
files: AppServer/google/appengine/ext/datastore_admin/main.py
files: {% extends "base.html" %}
  {% block title %}Confirm Import of backup information{% endblock %}
  {% block body %}
    <h2>Datastore Admin: Import backup information</h2>
    {% if error %}
      <div class="ae-errorbox">{{ error|escape }}</div>
      <a href="{{ datastore_admin_home }}"><button>Back</button></a>
    {% else %}
      {% if not other_backup_info_files and not selected_backup_info_file %}
        <div class="ae-errorbox">No backup information files were found.</div>
        <a href="{{ datastore_admin_home }}"><button>Back</button></a>
      {% else %}
        <form action="{{base_path}}/{{form_target}}" method="post" style="width:39.39em;">
          <input type="hidden" name="xsrf_token" value="{{xsrf_token|escape}}">
          {% if not other_backup_info_files %}
            <b>Confirm import of:</b>
            <input type="hidden" name="gs_handle" value="{{selected_backup_info_file}}">{{selected_backup_info_file|escape}}
          {% else %}
            {% if selected_backup_info_file %}
              <b>Confirm import of:</b><br/>
              <input type="radio" name="gs_handle" value="{{selected_backup_info_file}}" checked>{{selected_backup_info_file|escape}}</br>
            {% else %}
              {% if backup_info_specified %}
                <b>Requested backup was not found, available backups:</b><br/>
              {% else %}
                <b>Select an available backup:</b><br/>
              {% endif %}
            {% endif %}
            {% for backup_info_file in other_backup_info_files %}
              <input type="radio" name="gs_handle" value="{{backup_info_file}}">{{backup_info_file|escape}}</br>
            {% endfor %}
          {% endif %}
          {% if run_as_a_service %}
            <input type="hidden" name="run_as_a_service" value="{{ run_as_a_service|escape }}">
          {% endif %}
          <input type="hidden" name="service_account_name" value="{{ service_account_name }}">
          <input type="Submit" name="Import" value="Add to backup list">
          <input type="Submit" name="Restore" value="Restore from backup">
          <a href="{{ datastore_admin_home }}">Cancel</a>
        </form>
      {% endif %}
    {% endif %}
  {% endblock %}
  
  
  
  [... the rest is omitted ...]
files: AppServer/google/appengine/ext/datastore_admin/templates/confirm_backup_import.html
files: {% extends "base.html" %}
  {% block title %}Confirm Import of backup information{% endblock %}
  {% block body %}
    <h2>Datastore Admin: Import backup information</h2>
    {% if error %}
      <div class="ae-errorbox">{{ error|escape }}</div>
      <a href="{{ datastore_admin_home }}"><button>Back</button></a>
    {% else %}
      {% if not other_backup_info_files and not selected_backup_info_file %}
        <div class="ae-errorbox">No backup information files were found.</div>
        <a href="{{ datastore_admin_home }}"><button>Back</button></a>
      {% else %}
        <form action="{{base_path}}/{{form_target}}" method="post" style="width:39.39em;">
          <input type="hidden" name="xsrf_token" value="{{xsrf_token|escape}}">
          {% if not other_backup_info_files %}
            <b>Confirm import of:</b>
            <input type="hidden" name="gs_handle" value="{{selected_backup_info_file}}">{{selected_backup_info_file|escape}}
          {% else %}
            {% if selected_backup_info_file %}
              <b>Confirm import of:</b><br/>
              <input type="radio" name="gs_handle" value="{{selected_backup_info_file}}" checked>{{selected_backup_info_file|escape}}</br>
            {% else %}
              {% if backup_info_specified %}
                <b>Requested backup was not found, available backups:</b><br/>
              {% else %}
                <b>Select an available backup:</b><br/>
              {% endif %}
            {% endif %}
            {% for backup_info_file in other_backup_info_files %}
              <input type="radio" name="gs_handle" value="{{backup_info_file}}">{{backup_info_file|escape}}</br>
            {% endfor %}
          {% endif %}
          {% if run_as_a_service %}
            <input type="hidden" name="run_as_a_service" value="{{ run_as_a_service|escape }}">
          {% endif %}
          <input type="Submit" name="Import" value="Add to backup list">
          <input type="Submit" name="Restore" value="Restore from backup">
          <a href="{{ datastore_admin_home }}">Cancel</a>
        </form>
      {% endif %}
    {% endif %}
  {% endblock %}
  
  
  
  [... the rest is omitted ...]
files: AppServer/google/appengine/ext/datastore_admin/templates/confirm_backup_import.html
files: {% extends "base.html" %}
  {% block title %}Confirm Restore from a backup copy{% endblock %}
  {% block body %}
    <h2>Datastore Admin: Restore from a backup copy</h2>
    {% if backup %}
      Are you sure you want to restore from {{ backup.name }}?
      <br/>Restore does not delete any new entities added since the backup.
      However, it will replace existing entities with the backed-up version for the following kinds:
      <form action="{{base_path}}/{{form_target}}" method="post" style="width:39.39em;">
        {% if run_as_a_service %}
          <input type="hidden" name="run_as_a_service" value="{{ run_as_a_service|escape }}">
        {% endif %}
        <input type="hidden" name="backup_id" value="{{backup.key}}">
        <input type="hidden" name="xsrf_token" value="{{xsrf_token|escape}}">
        <input type="hidden" name="delete_backup_after_restore" value="{{ delete_backup_after_restore|escape }}">
        <ul>
          {% for kind in backup.kinds %}
            <li><input type="checkbox" name="kind" value="{{kind}}" checked="checked"/>{{ kind }}</li>
          {% endfor %}
        </ul>
        {% if notreadonly_warning %}
        <div>
          <h4 style="padding:0;margin-top:1em;margin-bottom:0.46em;">WARNING</h4>
          {% if original_app_warning %}
            <ul><li>
          {% else %}
            <p>
          {% endif %}
          The application is not in read-only mode and restore may
          conflict with other changes.
          To place the application in read-only mode,
          please follow the instructions at
          <a href="https://developers.google.com/appengine/docs/adminconsole/applicationsettings#Disable_Datastore_Writes" target="_blank">
          Disable Datastore Writes</a>
          {% if not original_app_warning %}
            </p>
            </div>
          {% endif %}
        {% endif %}
        {% if original_app_warning %}
          {% if not notreadonly_warning %}
            <div>
            <h4 style="padding:0;margin-top:1em;margin-bottom:0.46em;">WARNING</h4>
            <p>
          {% else %}
            <li>
          {% endif %}
          The backup data comes from a different application, {{original_app_warning|escape}}.
          {% if not notreadonly_warning %}
            </p>
          {% else %}
            </li>
          {% endif %}
          </div>
        {% endif %}
        <p>Operation will be performed using the selected queue:
          {% if queues %}
          <select name="queue">
            {% for queue in queues %}
              <option value="{{ queue }}">{{ queue }}</option>
            {% endfor %}
          </select>
          {% else %}
          <input type="text" name="queue" value="default"/>
          {% endif %}
        </p>
        <p>
          <label for="service_account_name">
            Service account name
            <img class="ae-help-icon" src="{{ base_path }}/static/img/help.gif" height="14" width="14" alt="help"
                 title="You can define additional service accounts using the dashboard.">
          </label>
          <select id="service_account_name" name="service_account_name">
            <option value="">--Choose an option--</option>
            {% for account_name in service_accounts %}
            <option value="{{ account_name }}">{{ account_name }}</option>
            {% endfor %}
          </select>
        </p>
        <input type="submit" value="Restore">
        <a href="{{ datastore_admin_home }}">Cancel</a>
      </form>
    {% else %}
      <div class="ae-errorbox">Backup was not found.</div>
    {% endif %}
  {% endblock %}
  
  
  [... the rest is omitted ...]
files: AppServer/google/appengine/ext/datastore_admin/templates/confirm_restore_from_backup.html
files: {% extends "base.html" %}
  {% block title %}Confirm Restore from a backup copy{% endblock %}
  {% block body %}
    <h2>Datastore Admin: Restore from a backup copy</h2>
    {% if backup %}
      Are you sure you want to restore from {{ backup.name }}?
      <br/>Restore does not delete any new entities added since the backup.
      However, it will replace existing entities with the backed-up version for the following kinds:
      <form action="{{base_path}}/{{form_target}}" method="post" style="width:39.39em;">
        {% if run_as_a_service %}
          <input type="hidden" name="run_as_a_service" value="{{ run_as_a_service|escape }}">
        {% endif %}
        <input type="hidden" name="backup_id" value="{{backup.key}}">
        <input type="hidden" name="xsrf_token" value="{{xsrf_token|escape}}">
        <input type="hidden" name="delete_backup_after_restore" value="{{ delete_backup_after_restore|escape }}">
        <ul>
          {% for kind in backup.kinds %}
            <li><input type="checkbox" name="kind" value="{{kind}}" checked="checked"/>{{ kind }}</li>
          {% endfor %}
        </ul>
        {% if notreadonly_warning %}
        <div>
          <h4 style="padding:0;margin-top:1em;margin-bottom:0.46em;">WARNING</h4>
          {% if original_app_warning %}
            <ul><li>
          {% else %}
            <p>
          {% endif %}
          The application is not in read-only mode and restore may
          conflict with other changes.
          To place the application in read-only mode,
          please follow the instructions at
          <a href="https://developers.google.com/appengine/docs/adminconsole/applicationsettings#Disable_Datastore_Writes" target="_blank">
          Disable Datastore Writes</a>
          {% if not original_app_warning %}
            </p>
            </div>
          {% endif %}
        {% endif %}
        {% if original_app_warning %}
          {% if not notreadonly_warning %}
            <div>
            <h4 style="padding:0;margin-top:1em;margin-bottom:0.46em;">WARNING</h4>
            <p>
          {% else %}
            <li>
          {% endif %}
          The backup data comes from a different application, {{original_app_warning|escape}}.
          {% if not notreadonly_warning %}
            </p>
          {% else %}
            </li>
          {% endif %}
          </div>
        {% endif %}
        <p>Operation will be performed using the selected queue:
          {% if queues %}
          <select name="queue">
            {% for queue in queues %}
              <option value="{{ queue }}">{{ queue }}</option>
            {% endfor %}
          </select>
          {% else %}
          <input type="text" name="queue" value="default"/>
          {% endif %}
        </p>
        <input type="submit" value="Restore">
        <a href="{{ datastore_admin_home }}">Cancel</a>
      </form>
    {% else %}
      <div class="ae-errorbox">Backup was not found.</div>
    {% endif %}
  {% endblock %}
  
  
  [... the rest is omitted ...]
files: AppServer/google/appengine/ext/datastore_admin/templates/confirm_restore_from_backup.html
files: {% extends "base.html" %}
  {% block body %}
    <h2>Datastore Admin of {{hosting_app_id}}</h2>
    {% if run_as_a_service or offer_service %}
      <div class="ae-promo ae-rounded-sml">
        <strong>
          The managed backup service is deprecated, and will no longer be available as of
          Mar 27th, 2018.
        </strong>
      </div>
    {% endif %}
    {% if run_as_a_service %}
      <div class="ae-message ae-rounded-sml">
        Backup/Restore running as a service.
        <a href="{{datastore_admin_home}}?service=T">Disable</a>
      </div>
    {% else %}
      {% if offer_service %}
        <div class="ae-message ae-rounded-sml">
          This application may run backup/restore as a service.
          <a href="{{datastore_admin_home}}?run_as_a_service=True">Enable</a>
        </div>
      {% endif %}
    {% endif %}
    {% if error %}
    <div class="ae-errorbox">
      {{ error|escape }}
    </div>
    {% endif %}
    <div class="ae-table-caption">
      <div class="ae-paginate">
        <div class="g-section g-tpl-50-50 g-split">
          <div class="g-unit g-first">
            <strong>Entities</strong>
          </div>
          <div class="g-unit">
            <span id="ae-dash-quota-refresh-info">
            {% if last_stats_update %}
              Entity statistics last updated
              {{ last_stats_update|date:"M j, Y g:i a T" }}
            {% else %}
              Entity statistics are temporarily unavailable and should appear within 48 hours.
            {% endif %}
            <a target="_blank"
               href="https://cloud.google.com/appengine/docs/standard/python/console/datastore-statistics">
              <img class="ae-help-icon" src="{{ base_path }}/static/img/help.gif"
                height="14" width="14" alt="help">
            </a>
            </span>
          </div>
        </div>
      </div>
    </div>
    {% if kind_stats %}
    <form action="{{ base_path }}/" method="post">
      {% if run_as_a_service %}
        <input type="hidden" name="run_as_a_service" value="{{ run_as_a_service|escape }}">
      {% endif %}
      {% if has_namespace %}
      <input type="hidden" name="namespace" value="{{ namespace|escape }}">
      {% endif %}
      <input type="hidden" name="app_id" value="{{ app_id|escape }}">
      <table id="kinds" class="ae-table ae-table-striped">
        <thead>
          <tr>
            <th scope="col" class="cbc tct-selectall"></th>
            <th scope="col" class="id">Entity Kind</th>
            <th scope="col" class="id"># Entities</th>
            <th scope="col" class="id">Avg. Size/Entity</th>
            <th scope="col" class="id">Entities Size</th>
            <th scope="col" class="id">Total Size</th>
          </tr>
        </thead>
        <tbody>
          {% for kind in kind_stats %}
            <tr>
              <td class="cbc">
                <input type="checkbox" name="kind" value="{{ kind.kind_name }}"
                       id="{{ kind.kind_name }}">
              </td>
              <td class="id">
                <label for="{{ kind.kind_name }}">{{ kind.kind_name }}</label>
              </td>
              {% if kind.count %}
                <td class="id">{{ kind.count }}</td>
                <td class="id">{{ kind.average_bytes_str }}</td>
                <td class="id">{{ kind.entity_bytes_str }}</td>
                <td class="id">{{ kind.total_bytes_str }}</td>
              {% else %}
                <td class="id" style="color: #666667;">Stats not available</td>
                <td class="id" style="color: #666667;">Stats not available</td>
                <td class="id" style="color: #666667;">Stats not available</td>
                <td class="id" style="color: #666667;">Stats not available</td>
              {% endif %}
            </tr>
          {% endfor %}
          {% if more_kinds %}
            <tr>
              <td colspan="6" class="cbc">
                Not all kinds could be displayed, possibly because there were too many. To back up a kind that does not appear in this list, use a <a href="https://developers.google.com/appengine/articles/scheduled_backups" target="_blank">scheduled backup</a>.
              </td>
            </tr>
          {% endif %}
        </tbody>
        <tfoot>
          <tr>
            <td colspan="6">
              {% for action in action_list %}
                <input type="submit" name="action"
                  class="goog-button" value="{{ action }}">
              {% endfor %}
            </td>
          </tr>
        </tfoot>
      </table>
    </form>
    {% else %}
      <table class="ae-table">
        <tbody>
          <tr>
            <td>
              <div class="ae-disabled"
                  style="padding-top: 0.75em; padding-bottom: 0.75em;
                         text-align: center;">
                Datastore is empty.
              </div>
            </td>
          </tr>
        </tbody>
      </table>
    {% endif %}
    {% if pending_backups %}
      <p>
      <div class="ae-table-caption">
        <div class="ae-paginate">
          <div class="g-section g-tpl-50-50 g-split">
            <div class="g-unit g-first">
              <strong>Pending Backups</strong>
            </div>
          </div>
        </div>
      </div>
      <form action="{{ base_path }}/" method="post">
        {% if run_as_a_service %}
          <input type="hidden" name="run_as_a_service" value="{{ run_as_a_service|escape }}">
        {% endif %}
        <input type="hidden" name="app_id" value="{{ app_id|escape }}">
        <table id="pending_backups" class="ae-table ae-table-striped">
          <thead>
            <tr>
              <th scope="col" class="cbc tct-selectall"></th>
              <th scope="col" class="id">Name</th>
              <th scope="col" class="id">Start-Time</th>
            </tr>
          </thead>
          <tbody>
            {% for pending_backup in pending_backups %}
              <tr>
                <td class="cbc">
                  <input type="checkbox" name="backup_id" value="{{ pending_backup.key }}" id="{{ pending_backup.key }}">
                </td>
                <td class="id">
                  <span title="Kinds: {{ pending_backup.kinds|join:', '|truncatewords:20|escape}}">{{ pending_backup.name }}{% if pending_backup.original_app %} (Imported from {{pending_backup.original_app|escape}}){% endif %}</span>
                </td>
                <td class="id">
                  {{ pending_backup.start_time }}
                </td>
              </tr>
            {% endfor %}
          </tbody>
          <tfoot>
            <tr>
              <td colspan="4" nowrap="nowrap">
                {% for pending_backup_action in pending_backup_action_list %}
                    <input type="submit" name="action" style="" class="goog-button" value="{{ pending_backup_action }}">
                {% endfor %}
              </td>
            </tr>
          </tfoot>
        </table>
      </form>
    {% endif %}
    <p>
    <div class="ae-table-caption">
      <div class="ae-paginate">
        <div class="g-section g-tpl-50-50 g-split">
          <div class="g-unit g-first">
            <strong>Backups</strong>
          </div>
        </div>
      </div>
    </div>
    {% if backups %}
      <form action="{{ base_path }}/" method="post">
        {% if run_as_a_service %}
          <input type="hidden" name="run_as_a_service" value="{{ run_as_a_service|escape }}">
        {% endif %}
        <input type="hidden" name="app_id" value="{{ app_id|escape }}">
        <table id="backups" class="ae-table ae-table-striped">
          <thead>
            <tr>
              <th scope="col" class="cbc tct-selectall"></th>
              <th scope="col" class="id">Name</th>
              <th scope="col" class="id">Start-Time</th>
              <th scope="col" class="id">End-Time</th>
            </tr>
          </thead>
          <tbody>
            {% for backup in backups %}
              <tr>
                <td class="cbc">
                  <input type="checkbox" name="backup_id" value="{{ backup.key }}" id="{{ backup.key }}">
                </td>
                <td class="id">
                  <span title="Kinds: {{ backup.kinds|join:', '|truncatewords:20|escape }}">{{ backup.name }}{% if backup.original_app %} (Imported from {{backup.original_app|escape}}){% endif %}</span>
                </td>
                <td class="id">
                  {{ backup.start_time }}
                </td>
                <td class="id">
                  {{ backup.complete_time }}
                </td>
              </tr>
            {% endfor %}
          </tbody>
          <tfoot>
            <tr>
              <td colspan="4" nowrap="nowrap">
                {% for backup_action in backup_action_list %}
                    <input type="submit" name="action" style="" class="goog-button" value="{{ backup_action }}">
                {% endfor %}
              </td>
            </tr>
          </tfoot>
        </table>
      </form>
    {% endif %}
    <div class="ae-table-caption">
      <form action="{{ base_path }}/" method="post" >
        {% if run_as_a_service %}
          <input type="hidden" name="run_as_a_service" value="{{ run_as_a_service|escape }}">
        {% endif %}
        <div style="display: block">
          <label for="service_account_name">
            Service account name
            <img class="ae-help-icon" src="{{ base_path }}/static/img/help.gif" height="14" width="14" alt="help"
                 title="You can define additional service accounts using the dashboard.">
          </label>
          <select id="service_account_name" name="service_account_name">
            <option value="">--Choose an option--</option>
            {% for account_name in service_accounts %}
            <option value="{{ account_name }}">{{ account_name }}</option>
            {% endfor %}
          </select>
        </div>
        <input type="text" name="gs_handle" value="" size="50" placeholder="Bucket or info file"
               title="Google Cloud Storage path of a backup info file or a bucket name"/>
        <input type="submit" name="action" value="Import Backup Information" title="Import Backup Information or restore from Google Cloud Storage"/>
      </form>
    </div>
    {% if active_operations %}
      <p>
      <div class="ae-table-caption">
        <div class="ae-paginate">
          <div class="g-section g-tpl-50-50 g-split">
            <div class="g-unit g-first">
              <strong>Active Operations</strong>
            </div>
          </div>
        </div>
      </div>
      <table id="operations" class="ae-table ae-table-striped">
        <thead>
          <tr>
            <th scope="col" class="id">Description</th>
            <th scope="col" class="id">Jobs</th>
          </tr>
        </thead>
        <tbody>
          {% for operation in active_operations %}
            <tr>
              <td class="id
  
  [... the rest is omitted ...]
files: AppServer/google/appengine/ext/datastore_admin/templates/list_actions.html
files: {% extends "base.html" %}
  {% block body %}
    <h2>Datastore Admin of {{hosting_app_id}}</h2>
    {% if run_as_a_service or offer_service %}
      <div class="ae-promo ae-rounded-sml">
        <strong>
          The managed backup service is deprecated, and will no longer be available as of
          Mar 27th, 2018.
        </strong>
      </div>
    {% endif %}
    {% if run_as_a_service %}
      <div class="ae-message ae-rounded-sml">
        Backup/Restore running as a service.
        <a href="{{datastore_admin_home}}?service=T">Disable</a>
      </div>
    {% else %}
      {% if offer_service %}
        <div class="ae-message ae-rounded-sml">
          This application may run backup/restore as a service.
          <a href="{{datastore_admin_home}}?run_as_a_service=True">Enable</a>
        </div>
      {% endif %}
    {% endif %}
    {% if error %}
    <div class="ae-errorbox">
      {{ error|escape }}
    </div>
    {% endif %}
    <div class="ae-table-caption">
      <div class="ae-paginate">
        <div class="g-section g-tpl-50-50 g-split">
          <div class="g-unit g-first">
            <strong>Entities</strong>
          </div>
          <div class="g-unit">
            <span id="ae-dash-quota-refresh-info">
            {% if last_stats_update %}
              Entity statistics last updated
              {{ last_stats_update|date:"M j, Y g:i a T" }}
            {% else %}
              Entity statistics are temporarily unavailable and should appear within 48 hours.
            {% endif %}
            <a target="_blank"
               href="https://cloud.google.com/appengine/docs/standard/python/console/datastore-statistics">
              <img class="ae-help-icon" src="{{ base_path }}/static/img/help.gif"
                height="14" width="14" alt="help">
            </a>
            </span>
          </div>
        </div>
      </div>
    </div>
    {% if kind_stats %}
    <form action="{{ base_path }}/" method="post">
      {% if run_as_a_service %}
        <input type="hidden" name="run_as_a_service" value="{{ run_as_a_service|escape }}">
      {% endif %}
      {% if has_namespace %}
      <input type="hidden" name="namespace" value="{{ namespace|escape }}">
      {% endif %}
      <input type="hidden" name="app_id" value="{{ app_id|escape }}">
      <table id="kinds" class="ae-table ae-table-striped">
        <thead>
          <tr>
            <th scope="col" class="cbc tct-selectall"></th>
            <th scope="col" class="id">Entity Kind</th>
            <th scope="col" class="id"># Entities</th>
            <th scope="col" class="id">Avg. Size/Entity</th>
            <th scope="col" class="id">Entities Size</th>
            <th scope="col" class="id">Total Size</th>
          </tr>
        </thead>
        <tbody>
          {% for kind in kind_stats %}
            <tr>
              <td class="cbc">
                <input type="checkbox" name="kind" value="{{ kind.kind_name }}"
                       id="{{ kind.kind_name }}">
              </td>
              <td class="id">
                <label for="{{ kind.kind_name }}">{{ kind.kind_name }}</label>
              </td>
              {% if kind.count %}
                <td class="id">{{ kind.count }}</td>
                <td class="id">{{ kind.average_bytes_str }}</td>
                <td class="id">{{ kind.entity_bytes_str }}</td>
                <td class="id">{{ kind.total_bytes_str }}</td>
              {% else %}
                <td class="id" style="color: #666667;">Stats not available</td>
                <td class="id" style="color: #666667;">Stats not available</td>
                <td class="id" style="color: #666667;">Stats not available</td>
                <td class="id" style="color: #666667;">Stats not available</td>
              {% endif %}
            </tr>
          {% endfor %}
          {% if more_kinds %}
            <tr>
              <td colspan="6" class="cbc">
                Not all kinds could be displayed, possibly because there were too many. To back up a kind that does not appear in this list, use a <a href="https://developers.google.com/appengine/articles/scheduled_backups" target="_blank">scheduled backup</a>.
              </td>
            </tr>
          {% endif %}
        </tbody>
        <tfoot>
          <tr>
            <td colspan="6">
              {% for action in action_list %}
                <input type="submit" name="action"
                  class="goog-button" value="{{ action }}">
              {% endfor %}
            </td>
          </tr>
        </tfoot>
      </table>
    </form>
    {% else %}
      <table class="ae-table">
        <tbody>
          <tr>
            <td>
              <div class="ae-disabled"
                  style="padding-top: 0.75em; padding-bottom: 0.75em;
                         text-align: center;">
                Datastore is empty.
              </div>
            </td>
          </tr>
        </tbody>
      </table>
    {% endif %}
    {% if pending_backups %}
      <p>
      <div class="ae-table-caption">
        <div class="ae-paginate">
          <div class="g-section g-tpl-50-50 g-split">
            <div class="g-unit g-first">
              <strong>Pending Backups</strong>
            </div>
          </div>
        </div>
      </div>
      <form action="{{ base_path }}/" method="post">
        {% if run_as_a_service %}
          <input type="hidden" name="run_as_a_service" value="{{ run_as_a_service|escape }}">
        {% endif %}
        <input type="hidden" name="app_id" value="{{ app_id|escape }}">
        <table id="pending_backups" class="ae-table ae-table-striped">
          <thead>
            <tr>
              <th scope="col" class="cbc tct-selectall"></th>
              <th scope="col" class="id">Name</th>
              <th scope="col" class="id">Start-Time</th>
            </tr>
          </thead>
          <tbody>
            {% for pending_backup in pending_backups %}
              <tr>
                <td class="cbc">
                  <input type="checkbox" name="backup_id" value="{{ pending_backup.key }}" id="{{ pending_backup.key }}">
                </td>
                <td class="id">
                  <span title="Kinds: {{ pending_backup.kinds|join:', '|truncatewords:20|escape}}">{{ pending_backup.name }}{% if pending_backup.original_app %} (Imported from {{pending_backup.original_app|escape}}){% endif %}</span>
                </td>
                <td class="id">
                  {{ pending_backup.start_time }}
                </td>
              </tr>
            {% endfor %}
          </tbody>
          <tfoot>
            <tr>
              <td colspan="4" nowrap="nowrap">
                {% for pending_backup_action in pending_backup_action_list %}
                    <input type="submit" name="action" style="" class="goog-button" value="{{ pending_backup_action }}">
                {% endfor %}
              </td>
            </tr>
          </tfoot>
        </table>
      </form>
    {% endif %}
    <p>
    <div class="ae-table-caption">
      <div class="ae-paginate">
        <div class="g-section g-tpl-50-50 g-split">
          <div class="g-unit g-first">
            <strong>Backups</strong>
          </div>
        </div>
      </div>
    </div>
    {% if backups %}
      <form action="{{ base_path }}/" method="post">
        {% if run_as_a_service %}
          <input type="hidden" name="run_as_a_service" value="{{ run_as_a_service|escape }}">
        {% endif %}
        <input type="hidden" name="app_id" value="{{ app_id|escape }}">
        <table id="backups" class="ae-table ae-table-striped">
          <thead>
            <tr>
              <th scope="col" class="cbc tct-selectall"></th>
              <th scope="col" class="id">Name</th>
              <th scope="col" class="id">Start-Time</th>
              <th scope="col" class="id">End-Time</th>
            </tr>
          </thead>
          <tbody>
            {% for backup in backups %}
              <tr>
                <td class="cbc">
                  <input type="checkbox" name="backup_id" value="{{ backup.key }}" id="{{ backup.key }}">
                </td>
                <td class="id">
                  <span title="Kinds: {{ backup.kinds|join:', '|truncatewords:20|escape }}">{{ backup.name }}{% if backup.original_app %} (Imported from {{backup.original_app|escape}}){% endif %}</span>
                </td>
                <td class="id">
                  {{ backup.start_time }}
                </td>
                <td class="id">
                  {{ backup.complete_time }}
                </td>
              </tr>
            {% endfor %}
          </tbody>
          <tfoot>
            <tr>
              <td colspan="4" nowrap="nowrap">
                {% for backup_action in backup_action_list %}
                    <input type="submit" name="action" style="" class="goog-button" value="{{ backup_action }}">
                {% endfor %}
              </td>
            </tr>
          </tfoot>
        </table>
      </form>
    {% endif %}
    <div class="ae-table-caption">
      <form action="{{ base_path }}/" method="post" >
        {% if run_as_a_service %}
          <input type="hidden" name="run_as_a_service" value="{{ run_as_a_service|escape }}">
        {% endif %}
        <input type="submit" name="action" value="Import Backup Information" title="Import Backup Information or restore from Google Cloud Storage"/>
        <input type="text" name="gs_handle" value="" size="50" title="Google Cloud Storage path of a backup info file or a bucket name"/>
      </form>
    </div>
    {% if active_operations %}
      <p>
      <div class="ae-table-caption">
        <div class="ae-paginate">
          <div class="g-section g-tpl-50-50 g-split">
            <div class="g-unit g-first">
              <strong>Active Operations</strong>
            </div>
          </div>
        </div>
      </div>
      <table id="operations" class="ae-table ae-table-striped">
        <thead>
          <tr>
            <th scope="col" class="id">Description</th>
            <th scope="col" class="id">Jobs</th>
          </tr>
        </thead>
        <tbody>
          {% for operation in active_operations %}
            <tr>
              <td class="id">
                {{ operation.description }}
              </td>
              <td class="id">
                ({{operation.completed_jobs}} steps completed,
                {{operation.active_jobs}} active)
                {% for job_id in operation.active_job_ids %}
                 <br/><a href="{{ map_reduce_path }}?mapreduce_id={{job_id}}">Job {{job_id}}</a>
                {% endfor %}
              </td>
            </tr>
          {% endfor %}
        </tbody>
      </table>
    {% endif %}
    {% if completed_operations %}
      <p>
      <div class="ae-table-caption">
        <div class="ae-paginate">
          <div class="g-section g
  
  [... the rest is omitted ...]
files: AppServer/google/appengine/ext/datastore_admin/templates/list_actions.html
files: #!/usr/bin/env python
  #
  # Copyright 2007 Google Inc.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  #
  
  
  
  
  """Used render templates for datastore admin."""
  
  
  
  import base64
  import datetime
  import json
  import logging
  import os
  import random
  import time
  
  # AppScale: Use bundled mapreduce library.
  from google.appengine.ext.mapreduce import context
  from google.appengine.ext.mapreduce import control
  from google.appengine.ext.mapreduce import model
  from google.appengine.ext.mapreduce import operation as mr_operation
  from google.appengine.ext.mapreduce import util
  
  from google.appengine.api import app_identity
  from google.appengine.api import datastore
  from google.appengine.api import datastore_errors
  from google.appengine.api import memcache
  from google.appengine.api import urlfetch
  from google.appengine.api import users
  from google.appengine.datastore import datastore_rpc
  from google.appengine.datastore import entity_pb
  from google.appengine.ext import db
  import webapp2 as webapp
  from google.appengine.ext.datastore_admin import config
  from google.appengine.ext.db import metadata
  from google.appengine.ext.db import stats
  from google.appengine.ext.webapp import _template
  from google.appengine.runtime import apiproxy_errors
  
  
  MEMCACHE_NAMESPACE = '_ah-datastore_admin'
  XSRF_VALIDITY_TIME = 600
  KINDS_AND_SIZES_VAR = 'kinds_and_sizes'
  MAPREDUCE_MIN_SHARDS = 8
  MAPREDUCE_DEFAULT_SHARDS = 32
  MAPREDUCE_MAX_SHARDS = 256
  RESERVE_KEY_POOL_MAX_SIZE = 1000
  
  DATASTORE_ADMIN_OPERATION_KIND = '_AE_DatastoreAdmin_Operation'
  BACKUP_INFORMATION_KIND = '_AE_Backup_Information'
  BACKUP_INFORMATION_FILES_KIND = '_AE_Backup_Information_Kind_Files'
  BACKUP_INFORMATION_KIND_TYPE_INFO = '_AE_Backup_Information_Kind_Type_Info'
  DATASTORE_ADMIN_KINDS = (DATASTORE_ADMIN_OPERATION_KIND,
                           BACKUP_INFORMATION_KIND,
                           BACKUP_INFORMATION_FILES_KIND,
                           BACKUP_INFORMATION_KIND_TYPE_INFO)
  
  
  MAX_RPCS = 10
  
  
  def IsKindNameVisible(kind_name):
    return not (kind_name.startswith('__') or
                kind_name in DATASTORE_ADMIN_KINDS or
                kind_name in model._MAP_REDUCE_KINDS)
  
  
  def RenderToResponse(handler, template_file, template_params):
    """Render the given template_file using template_vals and write to response.
  
    Args:
      handler: the handler whose response we should render to
      template_file: the file name only of the template file we are using
      template_params: the parameters used to render the given template
    """
    template_params = _GetDefaultParams(template_params)
  
  
  
  
  
  
  
  
    handler.response.headers['X-FRAME-OPTIONS'] = ('ALLOW-FROM %s' %
                                                   config.ADMIN_CONSOLE_URL)
    template_params['admin_console_url'] = config.ADMIN_CONSOLE_URL
  
    rendered = _template.render(_GetTemplatePath(template_file), template_params)
    handler.response.out.write(rendered)
  
  
  def _GetTemplatePath(template_file):
    """Return the expected path for the template to render.
  
    Args:
      template_file: simple file name of template to render.
  
    Returns:
      path of template to render.
    """
    return os.path.join(
        os.path.dirname(__file__), 'templates', template_file)
  
  
  def _GetDefaultParams(template_params):
    """Update template_params to always contain necessary paths and never None."""
    if not template_params:
      template_params = {}
    template_params.update({
        'base_path': config.BASE_PATH,
        'mapreduce_path': config.MAPREDUCE_PATH,
    })
    return template_params
  
  
  def CreateXsrfToken(action):
    """Generate a token to be passed with a form for XSRF protection.
  
    Args:
      action: action to restrict token to
  
    Returns:
      suitably random token which is only valid for ten minutes and, if the user
      is authenticated, is only valid for the user that generated it.
    """
    user_str = _MakeUserStr()
  
    token = base64.b64encode(
        ''.join(chr(int(random.random()*255)) for _ in range(0, 64)))
  
    memcache.set(token,
                 (user_str, action),
                 time=XSRF_VALIDITY_TIME,
                 namespace=MEMCACHE_NAMESPACE)
  
    return token
  
  
  def ValidateXsrfToken(token, action):
    """Validate a given XSRF token by retrieving it from memcache.
  
    If the token has not been evicted from memcache (past ten minutes) and the
    user strings are equal, then this is a valid token.
  
    Args:
      token: token to validate from memcache.
      action: action that token should correspond to
  
    Returns:
      True if the token exists in memcache and the user strings are equal,
      False otherwise.
    """
    user_str = _MakeUserStr()
    token_obj = memcache.get(token, namespace=MEMCACHE_NAMESPACE)
  
    if not token_obj:
      return False
  
    token_str, token_action = token_obj
    if user_str != token_str or action != token_action:
      return False
  
    return True
  
  
  def CacheStats(formatted_results):
    """Cache last retrieved kind size values in memcache.
  
    Args:
      formatted_results: list of dictionaries of the form returnned by
        main._PresentableKindStats.
    """
    kinds_and_sizes = dict((kind['kind_name'], kind['total_bytes'])
                           for kind in formatted_results)
  
    memcache.set(KINDS_AND_SIZES_VAR,
                 kinds_and_sizes,
                 namespace=MEMCACHE_NAMESPACE)
  
  
  def RetrieveCachedStats():
    """Retrieve cached kind sizes from last datastore stats call.
  
    Returns:
      Dictionary mapping kind names to total bytes.
    """
    return memcache.get(KINDS_AND_SIZES_VAR, namespace=MEMCACHE_NAMESPACE)
  
  
  def _MakeUserStr():
    """Make a user string to use to represent the user.  'noauth' by default."""
    user = users.get_current_user()
    return user.nickname() if user else 'noauth'
  
  
  def GetPrettyBytes(bytes_num, significant_digits=0):
    """Get a pretty print view of the given number of bytes.
  
    This will give a string like 'X MBytes'.
  
    Args:
      bytes_num: the original number of bytes to pretty print.
      significant_digits: number of digits to display after the decimal point.
  
    Returns:
      A string that has the pretty print version of the given bytes.
      If bytes_num is to big the string 'Alot' will be returned.
    """
    byte_prefixes = ['', 'K', 'M', 'G', 'T', 'P', 'E']
    for i in range(0, 7):
      exp = i * 10
      if bytes_num < 1<<(exp + 10):
        if i == 0:
          formatted_bytes = str(bytes_num)
        else:
          formatted_bytes = '%.*f' % (significant_digits,
                                      (bytes_num * 1.0 / (1<<exp)))
        if formatted_bytes != '1':
          plural = 's'
        else:
          plural = ''
        return '%s %sByte%s' % (formatted_bytes, byte_prefixes[i], plural)
  
    logging.error('Number too high to convert: %d', bytes_num)
    return 'Alot'
  
  
  def FormatThousands(value):
    """Format a numerical value, inserting commas as thousands separators.
  
    Args:
      value: An integer, float, or string representation thereof.
        If the argument is a float, it is converted to a string using '%.2f'.
  
    Returns:
      A string with groups of 3 digits before the decimal point (if any)
      separated by commas.
  
    NOTE: We don't deal with whitespace, and we don't insert
    commas into long strings of digits after the decimal point.
    """
    if isinstance(value, float):
      value = '%.2f' % value
    else:
      value = str(value)
    if '.' in value:
      head, tail = value.split('.', 1)
      tail = '.' + tail
    elif 'e' in value:
      head, tail = value.split('e', 1)
      tail = 'e' + tail
    else:
      head = value
      tail = ''
    sign = ''
    if head.startswith('-'):
      sign = '-'
      head = head[1:]
    while len(head) > 3:
      tail = ',' + head[-3:] + tail
      head = head[:-3]
    return sign + head + tail
  
  
  def TruncDelta(delta):
    """Strips microseconds from a timedelta."""
    return datetime.timedelta(days=delta.days, seconds=delta.seconds)
  
  
  def GetPrintableStrs(namespace, kinds):
    """Returns tuples describing affected kinds and namespace.
  
    Args:
      namespace: namespace being targeted.
      kinds: list of kinds being targeted.
  
    Returns:
      (namespace_str, kind_str) tuple used for display to user.
    """
    namespace_str = namespace or ''
    if kinds:
      kind_str = 'all %s entities' % ', '.join(kinds)
    else:
      kind_str = ''
    return (namespace_str, kind_str)
  
  
  def ParseKindsAndSizes(kinds):
    """Parses kind|size list and returns template parameters.
  
    Args:
      kinds: list of kinds to process.
  
    Returns:
      sizes_known: whether or not all kind objects have known sizes.
      size_total: total size of objects with known sizes.
      len(kinds) - 2: for template rendering of greater than 3 kinds.
    """
    sizes_known = True
    size_total = 0
    kinds_and_sizes = RetrieveCachedStats()
  
    if kinds_and_sizes:
      for kind in kinds:
        if kind in kinds_and_sizes:
          size_total += kinds_and_sizes[kind]
        else:
          sizes_known = False
    else:
      sizes_known = False
  
    if size_total:
      size_total = GetPrettyBytes(size_total)
  
    return sizes_known, size_total, len(kinds) - 2
  
  
  def _CreateDatastoreConfig():
    """Create datastore config for use during datastore operations."""
    return datastore_rpc.Configuration(force_writes=True, deadline=60)
  
  
  def GenerateHomeUrl(unused_request):
    """Generates a link to the Datastore Admin main page.
  
    Primarily intended to be used for cancel buttons or links on error pages. To
    avoid any XSS security vulnerabilities the URL should not use any
    user-defined strings (unless proper precautions are taken).
  
    Args:
      unused_request: the webapp.Request object (to determine if certain query
        parameters need to be used).
  
    Returns:
      domain-relative URL for the main Datastore Admin page.
    """
    return config.BASE_PATH
  
  
  class MapreduceDoneHandler(webapp.RequestHandler):
    """Handler to delete data associated with successful MapReduce jobs."""
  
    SUFFIX = 'mapreduce_done'
  
    def post(self):
      """Mapreduce done callback to delete job data if it was successful."""
      if 'Mapreduce-Id' in self.request.headers:
        mapreduce_id = self.request.headers['Mapreduce-Id']
        mapreduce_state = model.MapreduceState.get_by_job_id(mapreduce_id)
        if not mapreduce_state:
          logging.error('Done
  
  [... the rest is omitted ...]
files: AppServer/google/appengine/ext/datastore_admin/utils.py
files: #!/usr/bin/env python
  #
  # Copyright 2007 Google Inc.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  #
  
  
  
  
  """Used render templates for datastore admin."""
  
  
  
  import base64
  import datetime
  import logging
  import os
  import random
  import time
  
  # AppScale: Use bundled mapreduce library.
  from google.appengine.ext.mapreduce import context
  from google.appengine.ext.mapreduce import control
  from google.appengine.ext.mapreduce import model
  from google.appengine.ext.mapreduce import operation as mr_operation
  from google.appengine.ext.mapreduce import util
  
  from google.appengine.api import datastore
  from google.appengine.api import datastore_errors
  from google.appengine.api import memcache
  from google.appengine.api import users
  from google.appengine.datastore import datastore_rpc
  from google.appengine.datastore import entity_pb
  from google.appengine.ext import db
  import webapp2 as webapp
  from google.appengine.ext.datastore_admin import config
  from google.appengine.ext.db import metadata
  from google.appengine.ext.db import stats
  from google.appengine.ext.webapp import _template
  from google.appengine.runtime import apiproxy_errors
  
  
  MEMCACHE_NAMESPACE = '_ah-datastore_admin'
  XSRF_VALIDITY_TIME = 600
  KINDS_AND_SIZES_VAR = 'kinds_and_sizes'
  MAPREDUCE_MIN_SHARDS = 8
  MAPREDUCE_DEFAULT_SHARDS = 32
  MAPREDUCE_MAX_SHARDS = 256
  RESERVE_KEY_POOL_MAX_SIZE = 1000
  
  DATASTORE_ADMIN_OPERATION_KIND = '_AE_DatastoreAdmin_Operation'
  BACKUP_INFORMATION_KIND = '_AE_Backup_Information'
  BACKUP_INFORMATION_FILES_KIND = '_AE_Backup_Information_Kind_Files'
  BACKUP_INFORMATION_KIND_TYPE_INFO = '_AE_Backup_Information_Kind_Type_Info'
  DATASTORE_ADMIN_KINDS = (DATASTORE_ADMIN_OPERATION_KIND,
                           BACKUP_INFORMATION_KIND,
                           BACKUP_INFORMATION_FILES_KIND,
                           BACKUP_INFORMATION_KIND_TYPE_INFO)
  
  
  MAX_RPCS = 10
  
  
  def IsKindNameVisible(kind_name):
    return not (kind_name.startswith('__') or
                kind_name in DATASTORE_ADMIN_KINDS or
                kind_name in model._MAP_REDUCE_KINDS)
  
  
  def RenderToResponse(handler, template_file, template_params):
    """Render the given template_file using template_vals and write to response.
  
    Args:
      handler: the handler whose response we should render to
      template_file: the file name only of the template file we are using
      template_params: the parameters used to render the given template
    """
    template_params = _GetDefaultParams(template_params)
  
  
  
  
  
  
  
  
    handler.response.headers['X-FRAME-OPTIONS'] = ('ALLOW-FROM %s' %
                                                   config.ADMIN_CONSOLE_URL)
    template_params['admin_console_url'] = config.ADMIN_CONSOLE_URL
  
    rendered = _template.render(_GetTemplatePath(template_file), template_params)
    handler.response.out.write(rendered)
  
  
  def _GetTemplatePath(template_file):
    """Return the expected path for the template to render.
  
    Args:
      template_file: simple file name of template to render.
  
    Returns:
      path of template to render.
    """
    return os.path.join(
        os.path.dirname(__file__), 'templates', template_file)
  
  
  def _GetDefaultParams(template_params):
    """Update template_params to always contain necessary paths and never None."""
    if not template_params:
      template_params = {}
    template_params.update({
        'base_path': config.BASE_PATH,
        'mapreduce_path': config.MAPREDUCE_PATH,
    })
    return template_params
  
  
  def CreateXsrfToken(action):
    """Generate a token to be passed with a form for XSRF protection.
  
    Args:
      action: action to restrict token to
  
    Returns:
      suitably random token which is only valid for ten minutes and, if the user
      is authenticated, is only valid for the user that generated it.
    """
    user_str = _MakeUserStr()
  
    token = base64.b64encode(
        ''.join(chr(int(random.random()*255)) for _ in range(0, 64)))
  
    memcache.set(token,
                 (user_str, action),
                 time=XSRF_VALIDITY_TIME,
                 namespace=MEMCACHE_NAMESPACE)
  
    return token
  
  
  def ValidateXsrfToken(token, action):
    """Validate a given XSRF token by retrieving it from memcache.
  
    If the token has not been evicted from memcache (past ten minutes) and the
    user strings are equal, then this is a valid token.
  
    Args:
      token: token to validate from memcache.
      action: action that token should correspond to
  
    Returns:
      True if the token exists in memcache and the user strings are equal,
      False otherwise.
    """
    user_str = _MakeUserStr()
    token_obj = memcache.get(token, namespace=MEMCACHE_NAMESPACE)
  
    if not token_obj:
      return False
  
    token_str, token_action = token_obj
    if user_str != token_str or action != token_action:
      return False
  
    return True
  
  
  def CacheStats(formatted_results):
    """Cache last retrieved kind size values in memcache.
  
    Args:
      formatted_results: list of dictionaries of the form returnned by
        main._PresentableKindStats.
    """
    kinds_and_sizes = dict((kind['kind_name'], kind['total_bytes'])
                           for kind in formatted_results)
  
    memcache.set(KINDS_AND_SIZES_VAR,
                 kinds_and_sizes,
                 namespace=MEMCACHE_NAMESPACE)
  
  
  def RetrieveCachedStats():
    """Retrieve cached kind sizes from last datastore stats call.
  
    Returns:
      Dictionary mapping kind names to total bytes.
    """
    return memcache.get(KINDS_AND_SIZES_VAR, namespace=MEMCACHE_NAMESPACE)
  
  
  def _MakeUserStr():
    """Make a user string to use to represent the user.  'noauth' by default."""
    user = users.get_current_user()
    return user.nickname() if user else 'noauth'
  
  
  def GetPrettyBytes(bytes_num, significant_digits=0):
    """Get a pretty print view of the given number of bytes.
  
    This will give a string like 'X MBytes'.
  
    Args:
      bytes_num: the original number of bytes to pretty print.
      significant_digits: number of digits to display after the decimal point.
  
    Returns:
      A string that has the pretty print version of the given bytes.
      If bytes_num is to big the string 'Alot' will be returned.
    """
    byte_prefixes = ['', 'K', 'M', 'G', 'T', 'P', 'E']
    for i in range(0, 7):
      exp = i * 10
      if bytes_num < 1<<(exp + 10):
        if i == 0:
          formatted_bytes = str(bytes_num)
        else:
          formatted_bytes = '%.*f' % (significant_digits,
                                      (bytes_num * 1.0 / (1<<exp)))
        if formatted_bytes != '1':
          plural = 's'
        else:
          plural = ''
        return '%s %sByte%s' % (formatted_bytes, byte_prefixes[i], plural)
  
    logging.error('Number too high to convert: %d', bytes_num)
    return 'Alot'
  
  
  def FormatThousands(value):
    """Format a numerical value, inserting commas as thousands separators.
  
    Args:
      value: An integer, float, or string representation thereof.
        If the argument is a float, it is converted to a string using '%.2f'.
  
    Returns:
      A string with groups of 3 digits before the decimal point (if any)
      separated by commas.
  
    NOTE: We don't deal with whitespace, and we don't insert
    commas into long strings of digits after the decimal point.
    """
    if isinstance(value, float):
      value = '%.2f' % value
    else:
      value = str(value)
    if '.' in value:
      head, tail = value.split('.', 1)
      tail = '.' + tail
    elif 'e' in value:
      head, tail = value.split('e', 1)
      tail = 'e' + tail
    else:
      head = value
      tail = ''
    sign = ''
    if head.startswith('-'):
      sign = '-'
      head = head[1:]
    while len(head) > 3:
      tail = ',' + head[-3:] + tail
      head = head[:-3]
    return sign + head + tail
  
  
  def TruncDelta(delta):
    """Strips microseconds from a timedelta."""
    return datetime.timedelta(days=delta.days, seconds=delta.seconds)
  
  
  def GetPrintableStrs(namespace, kinds):
    """Returns tuples describing affected kinds and namespace.
  
    Args:
      namespace: namespace being targeted.
      kinds: list of kinds being targeted.
  
    Returns:
      (namespace_str, kind_str) tuple used for display to user.
    """
    namespace_str = namespace or ''
    if kinds:
      kind_str = 'all %s entities' % ', '.join(kinds)
    else:
      kind_str = ''
    return (namespace_str, kind_str)
  
  
  def ParseKindsAndSizes(kinds):
    """Parses kind|size list and returns template parameters.
  
    Args:
      kinds: list of kinds to process.
  
    Returns:
      sizes_known: whether or not all kind objects have known sizes.
      size_total: total size of objects with known sizes.
      len(kinds) - 2: for template rendering of greater than 3 kinds.
    """
    sizes_known = True
    size_total = 0
    kinds_and_sizes = RetrieveCachedStats()
  
    if kinds_and_sizes:
      for kind in kinds:
        if kind in kinds_and_sizes:
          size_total += kinds_and_sizes[kind]
        else:
          sizes_known = False
    else:
      sizes_known = False
  
    if size_total:
      size_total = GetPrettyBytes(size_total)
  
    return sizes_known, size_total, len(kinds) - 2
  
  
  def _CreateDatastoreConfig():
    """Create datastore config for use during datastore operations."""
    return datastore_rpc.Configuration(force_writes=True, deadline=60)
  
  
  def GenerateHomeUrl(unused_request):
    """Generates a link to the Datastore Admin main page.
  
    Primarily intended to be used for cancel buttons or links on error pages. To
    avoid any XSS security vulnerabilities the URL should not use any
    user-defined strings (unless proper precautions are taken).
  
    Args:
      unused_request: the webapp.Request object (to determine if certain query
        parameters need to be used).
  
    Returns:
      domain-relative URL for the main Datastore Admin page.
    """
    return config.BASE_PATH
  
  
  class MapreduceDoneHandler(webapp.RequestHandler):
    """Handler to delete data associated with successful MapReduce jobs."""
  
    SUFFIX = 'mapreduce_done'
  
    def post(self):
      """Mapreduce done callback to delete job data if it was successful."""
      if 'Mapreduce-Id' in self.request.headers:
        mapreduce_id = self.request.headers['Mapreduce-Id']
        mapreduce_state = model.MapreduceState.get_by_job_id(mapreduce_id)
        if not mapreduce_state:
          logging.error('Done callback for no longer valid Mapreduce Id: %s',
                        mapreduce_id)
          return
  
        map
  
  [... the rest is omitted ...]
files: AppServer/google/appengine/ext/datastore_admin/utils.py