files: #!/usr/bin/env python
  #
  # Copyright 2007 Google Inc.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  #
  
  
  
  
  
  """Handler for data backup operation.
  
  Generic datastore admin console transfers control to ConfirmBackupHandler
  after selection of entities. The ConfirmBackupHandler confirms with user
  his choice, enters a backup name and transfers control to
  DoBackupHandler. DoBackupHandler starts backup mappers and displays confirmation
  page.
  
  This module also contains actual mapper code for backing data over.
  """
  
  
  
  
  
  
  
  
  from __future__ import with_statement
  
  
  
  import copy
  import cStringIO
  import datetime
  import itertools
  import logging
  import os
  import random
  import re
  import time
  import urllib
  import xml.dom.minidom
  
  from google.appengine._internal import cloudstorage
  
  # AppScale: Use bundled mapreduce library.
  from google.appengine.ext.mapreduce import context
  from google.appengine.ext.mapreduce import datastore_range_iterators as db_iters
  from google.appengine.ext.mapreduce import input_readers
  from google.appengine.ext.mapreduce import json_util
  from google.appengine.ext.mapreduce import operation as op
  from google.appengine.ext.mapreduce import output_writers
  from google.appengine.ext.mapreduce import parameters
  from google.appengine.ext.mapreduce import records
  
  from google.appengine.api import apiproxy_stub_map
  from google.appengine.api import app_identity
  from google.appengine.api import blobstore as blobstore_api
  from google.appengine.api import capabilities
  from google.appengine.api import datastore
  from google.appengine.api import datastore_errors
  from google.appengine.api import datastore_types
  from google.appengine.api import taskqueue
  from google.appengine.api import urlfetch
  from google.appengine.api.taskqueue import taskqueue_service_pb
  from google.appengine.datastore import entity_pb
  from google.appengine.ext import blobstore
  from google.appengine.ext import db
  from google.appengine.ext import deferred
  import webapp2 as webapp
  from google.appengine.runtime import apiproxy_errors
  from google.appengine.ext.datastore_admin import backup_pb2
  from google.appengine.ext.datastore_admin import config
  from google.appengine.ext.datastore_admin import utils
  
  
  XSRF_ACTION = 'backup'
  BUCKET_PATTERN = (r'^([a-zA-Z0-9]+([\-_]+[a-zA-Z0-9]+)*)'
                    r'(\.([a-zA-Z0-9]+([\-_]+[a-zA-Z0-9]+)*))*$')
  MAX_BUCKET_LEN = 222
  MIN_BUCKET_LEN = 3
  MAX_BUCKET_SEGMENT_LEN = 63
  NUM_KINDS_DEFERRED_THRESHOLD = 10
  MAX_BLOBS_PER_DELETE = 500
  TEST_WRITE_FILENAME_PREFIX = 'datastore_backup_write_test'
  MAX_KEYS_LIST_SIZE = 100
  MAX_TEST_FILENAME_TRIES = 10
  
  MEANING_TO_PRIMITIVE_TYPE = {
      entity_pb.Property.GD_WHEN: backup_pb2.EntitySchema.DATE_TIME,
      entity_pb.Property.GD_RATING: backup_pb2.EntitySchema.RATING,
      entity_pb.Property.ATOM_LINK: backup_pb2.EntitySchema.LINK,
      entity_pb.Property.ATOM_CATEGORY: backup_pb2.EntitySchema.CATEGORY,
      entity_pb.Property.GD_PHONENUMBER: backup_pb2.EntitySchema.PHONE_NUMBER,
      entity_pb.Property.GD_POSTALADDRESS: backup_pb2.EntitySchema.POSTAL_ADDRESS,
      entity_pb.Property.GD_EMAIL: backup_pb2.EntitySchema.EMAIL,
      entity_pb.Property.GD_IM: backup_pb2.EntitySchema.IM_HANDLE,
      entity_pb.Property.BLOBKEY: backup_pb2.EntitySchema.BLOB_KEY,
      entity_pb.Property.TEXT: backup_pb2.EntitySchema.TEXT,
      entity_pb.Property.BLOB: backup_pb2.EntitySchema.BLOB,
      entity_pb.Property.BYTESTRING: backup_pb2.EntitySchema.SHORT_BLOB
  }
  
  
  
  
  FILES_API_GS_FILESYSTEM = 'gs'
  FILES_API_BLOBSTORE_FILESYSTEM = 'blobstore'
  FILES_API_BLOBSTORE_PREFIX = '/blobstore/'
  
  BLOBSTORE_BACKUP_DISABLED_FEATURE_NAME = 'DisableBlobstoreBackups'
  
  BLOBSTORE_BACKUP_DISABLED_ERROR_MSG = (
      'Backups to blobstore are disabled, see '
      'https://cloud.google.com/appengine/docs/deprecations/blobstore_backups')
  
  BACKUP_RESTORE_HANDLER = __name__ + '.RestoreEntity.map'
  
  
  def _get_gcs_path_prefix_from_params_dict(params):
    """Returs the gcs_path_prefix from request or mapreduce dict.
  
    Args:
      params: A dict of parameters.
  
    Returns:
      A string containing GCS prefix for the backup. It can be either just the
      bucket or a bucket with a path (as present in the dictionary). Return None
      if 'gs_bucket_name' is not present in the dictionary.
    """
  
  
  
  
  
    return params.get('gs_bucket_name')
  
  
  def _get_basic_mapper_params(handler):
    namespace = handler.request.get('namespace', None)
    if namespace == '*':
      namespace = None
    return {'namespace': namespace}
  
  
  class GCSUtil(object):
    """Wrappers for cloudstorage methods to deal with /gs/ prefix.
  
    Files
  
  [... the rest is omitted ...]
files: AppServer/google/appengine/ext/datastore_admin/backup_handler.py
files: #!/usr/bin/env python
  #
  # Copyright 2007 Google Inc.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  #
  
  
  
  
  
  """Handler for data backup operation.
  
  Generic datastore admin console transfers control to ConfirmBackupHandler
  after selection of entities. The ConfirmBackupHandler confirms with user
  his choice, enters a backup name and transfers control to
  DoBackupHandler. DoBackupHandler starts backup mappers and displays confirmation
  page.
  
  This module also contains actual mapper code for backing data over.
  """
  
  
  
  
  
  
  
  
  from __future__ import with_statement
  
  
  
  import copy
  import cStringIO
  import datetime
  import itertools
  import json
  import logging
  import os
  import random
  import re
  import time
  import urllib
  import xml.dom.minidom
  
  from google.appengine._internal import cloudstorage
  
  # AppScale: Use bundled mapreduce library.
  from google.appengine.ext.mapreduce import context
  from google.appengine.ext.mapreduce import datastore_range_iterators as db_iters
  from google.appengine.ext.mapreduce import input_readers
  from google.appengine.ext.mapreduce import json_util
  from google.appengine.ext.mapreduce import operation as op
  from google.appengine.ext.mapreduce import output_writers
  from google.appengine.ext.mapreduce import parameters
  from google.appengine.ext.mapreduce import records
  
  from google.appengine.api import apiproxy_stub_map
  from google.appengine.api import app_identity
  from google.appengine.api import blobstore as blobstore_api
  from google.appengine.api import capabilities
  from google.appengine.api import datastore
  from google.appengine.api import datastore_errors
  from google.appengine.api import datastore_types
  from google.appengine.api import taskqueue
  from google.appengine.api import urlfetch
  from google.appengine.api.taskqueue import taskqueue_service_pb
  from google.appengine.datastore import entity_pb
  from google.appengine.ext import blobstore
  from google.appengine.ext import db
  from google.appengine.ext import deferred
  import webapp2 as webapp
  from google.appengine.runtime import apiproxy_errors
  from google.appengine.ext.datastore_admin import backup_pb2
  from google.appengine.ext.datastore_admin import config
  from google.appengine.ext.datastore_admin import utils
  
  
  XSRF_ACTION = 'backup'
  BUCKET_PATTERN = (r'^([a-zA-Z0-9]+([\-_]+[a-zA-Z0-9]+)*)'
                    r'(\.([a-zA-Z0-9]+([\-_]+[a-zA-Z0-9]+)*))*$')
  MAX_BUCKET_LEN = 222
  MIN_BUCKET_LEN = 3
  MAX_BUCKET_SEGMENT_LEN = 63
  NUM_KINDS_DEFERRED_THRESHOLD = 10
  MAX_BLOBS_PER_DELETE = 500
  TEST_WRITE_FILENAME_PREFIX = 'datastore_backup_write_test'
  MAX_KEYS_LIST_SIZE = 100
  MAX_TEST_FILENAME_TRIES = 10
  
  MEANING_TO_PRIMITIVE_TYPE = {
      entity_pb.Property.GD_WHEN: backup_pb2.EntitySchema.DATE_TIME,
      entity_pb.Property.GD_RATING: backup_pb2.EntitySchema.RATING,
      entity_pb.Property.ATOM_LINK: backup_pb2.EntitySchema.LINK,
      entity_pb.Property.ATOM_CATEGORY: backup_pb2.EntitySchema.CATEGORY,
      entity_pb.Property.GD_PHONENUMBER: backup_pb2.EntitySchema.PHONE_NUMBER,
      entity_pb.Property.GD_POSTALADDRESS: backup_pb2.EntitySchema.POSTAL_ADDRESS,
      entity_pb.Property.GD_EMAIL: backup_pb2.EntitySchema.EMAIL,
      entity_pb.Property.GD_IM: backup_pb2.EntitySchema.IM_HANDLE,
      entity_pb.Property.BLOBKEY: backup_pb2.EntitySchema.BLOB_KEY,
      entity_pb.Property.TEXT: backup_pb2.EntitySchema.TEXT,
      entity_pb.Property.BLOB: backup_pb2.EntitySchema.BLOB,
      entity_pb.Property.BYTESTRING: backup_pb2.EntitySchema.SHORT_BLOB
  }
  
  
  
  
  FILES_API_GS_FILESYSTEM = 'gs'
  FILES_API_BLOBSTORE_FILESYSTEM = 'blobstore'
  FILES_API_BLOBSTORE_PREFIX = '/blobstore/'
  
  BLOBSTORE_BACKUP_DISABLED_FEATURE_NAME = 'DisableBlobstoreBackups'
  
  BLOBSTORE_BACKUP_DISABLED_ERROR_MSG = (
      'Backups to blobstore are disabled, see '
      'https://cloud.google.com/appengine/docs/deprecations/blobstore_backups')
  
  BACKUP_RESTORE_HANDLER = __name__ + '.RestoreEntity.map'
  
  
  def get_service_account_names():
    """ AppScale: Fetch list of service accounts from IAM API. """
    project_id = app_identity.get_application_id()
    iam_location = 'https://127.0.0.1:17441'
    url = iam_location + '/v1/projects/{}/serviceAccounts'.format(project_id)
    token = app_identity.get_access_token(
        ['https://www.googleapis.com/auth/cloud-platform'])[0]
    headers = {'Authorization': 'Bearer {}'.format(token)}
    response = urlfetch.fetch(url, headers=headers, validate_certificate=False)
    try:
      accounts = json.loads(response.content)['accounts']
    except (KeyError, ValueError):
      raise ValueError('Invalid list of service accounts: '
                       '{}
  
  [... the rest is omitted ...]
files: AppServer/google/appengine/ext/datastore_admin/backup_handler.py
files: #!/usr/bin/env python
  #
  # Copyright 2007 Google Inc.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  #
  
  
  
  
  """Main module for datastore admin receiver.
  
  To use, add this to app.yaml:
    builtins:
    - datastore_admin: on
  """
  
  
  
  import logging
  import operator
  import os
  
  from google.appengine.api import app_identity
  from google.appengine.api import datastore_errors
  from google.appengine.api import users
  from google.appengine.ext import deferred
  import webapp2 as webapp
  from google.appengine.ext.db import stats
  from google.appengine.ext.webapp import util
  from google.appengine.runtime import apiproxy_errors
  from google.appengine.ext.datastore_admin import backup_handler
  from google.appengine.ext.datastore_admin import config
  from google.appengine.ext.datastore_admin import delete_handler
  from google.appengine.ext.datastore_admin import utils
  
  
  
  
  
  
  ENTITY_ACTIONS = {
      'Delete Entities': delete_handler.ConfirmDeleteHandler.Render,
      'Backup Entities': backup_handler.ConfirmBackupHandler.Render,
  }
  
  
  BACKUP_ACTIONS = {
      'Delete': backup_handler.ConfirmDeleteBackupHandler.Render,
      'Restore': backup_handler.ConfirmRestoreFromBackupHandler.Render,
      'Info': backup_handler.BackupInformationHandler.Render,
  }
  
  PENDING_BACKUP_ACTIONS = {
      'Abort': backup_handler.ConfirmAbortBackupHandler.Render,
      'Info': backup_handler.BackupInformationHandler.Render,
  }
  
  GET_ACTIONS = ENTITY_ACTIONS.copy()
  GET_ACTIONS.update(BACKUP_ACTIONS)
  GET_ACTIONS.update(PENDING_BACKUP_ACTIONS)
  GET_ACTIONS.update({'Import Backup Information':
                      backup_handler.ConfirmBackupImportHandler.Render})
  
  
  def _GetDatastoreStats(kinds_list, use_stats_kinds=False):
    """Retrieves stats for kinds.
  
    Args:
      kinds_list: List of known kinds.
      use_stats_kinds: If stats are available, kinds_list will be ignored and
        all kinds found in stats will be used instead.
  
    Returns:
      timestamp: records time that statistics were last updated.
      kind_dict: dictionary of kind objects with the following members:
      - kind_name: the name of this kind.
      - count: number of known entities of this type.
      - total_bytes_str: total bytes for this kind as a string.
      - average_bytes_str: average bytes per entity as a string.
    """
    global_stat = stats.GlobalStat.all().fetch(1)
    if not global_stat:
      return _KindsListToTuple(kinds_list)
  
    global_ts = global_stat[0].timestamp
  
    kind_stats = stats.KindStat.all().filter('timestamp =', global_ts).fetch(1000)
    if not kind_stats:
      return _KindsListToTuple(kinds_list)
  
    results = {}
    for kind_ent in kind_stats:
  
  
  
      if (not kind_ent.kind_name.startswith('__')
          and (use_stats_kinds or kind_ent.kind_name in kinds_list)
          and kind_ent.count > 0):
        results[kind_ent.kind_name] = _PresentatableKindStats(kind_ent)
  
    utils.CacheStats(results.values())
  
  
  
    for kind_str in kinds_list or []:
      if kind_str not in results:
        results[kind_str] = {'kind_name': kind_str}
  
    return (global_ts,
            sorted(results.values(), key=lambda x: x['kind_name']))
  
  
  def _KindsListToTuple(kinds_list):
    """Build default tuple when no datastore statistics are available. """
    return '', [{'kind_name': kind} for kind in sorted(kinds_list)]
  
  
  def _PresentatableKindStats(kind_ent):
    """Generate dict of presentable values for template."""
    count = kind_ent.count
    entity_bytes = kind_ent.entity_bytes
    total_bytes = kind_ent.bytes
    average_bytes = entity_bytes / count
    return {'kind_name': kind_ent.kind_name,
            'count': utils.FormatThousands(kind_ent.count),
            'entity_bytes_str': utils.GetPrettyBytes(entity_bytes),
            'entity_bytes': entity_bytes,
            'total_bytes_str': utils.GetPrettyBytes(total_bytes),
            'total_bytes': total_bytes,
            'average_bytes_str': utils.GetPrettyBytes(average_bytes),
           }
  
  
  class RouteByActionHandler(webapp.RequestHandler):
    """Route to the appropriate handler based on the action parameter."""
  
    def ListActions(self, error=None):
      """Handler for get requests to datastore_admin/confirm_delete."""
      use_stats_kinds = False
      kinds = []
      more_kinds = False
      try:
        kinds, more_kinds = utils.GetKinds()
        if not kinds:
          use_stats_kinds = True
          logging.warning('Found no kinds. Using datastore stats instead.')
      except datastore_errors.Error, e:
        logging.exception(e)
        use_stats_kinds = True
  
      last_stats_update, kind_stats = _GetDatastoreStats(
          kinds, use_stats_kinds=use_stats_kinds)
  
      template_params = {
          'run_as_a_service': self.request.get('run_as_a_service'),
          'datastore_admin_home': utils.GenerateHomeUrl(None),
          'offer_service': (self.request.get('service') and not
                            self.request.get('run_as_a_service')),
          'kind_stats': kind_stats,
          'more_kinds': more_kinds,
          'last_stats_update': last_stats_update,
          'app_id': self.request.get('app_id'),
  
  
  [... the rest is omitted ...]
files: AppServer/google/appengine/ext/datastore_admin/main.py
files: #!/usr/bin/env python
  #
  # Copyright 2007 Google Inc.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  #
  
  
  
  
  """Main module for datastore admin receiver.
  
  To use, add this to app.yaml:
    builtins:
    - datastore_admin: on
  """
  
  
  
  import logging
  import operator
  import os
  
  from google.appengine.api import app_identity
  from google.appengine.api import datastore_errors
  from google.appengine.api import users
  from google.appengine.ext import deferred
  import webapp2 as webapp
  from google.appengine.ext.db import stats
  from google.appengine.ext.webapp import util
  from google.appengine.runtime import apiproxy_errors
  from google.appengine.ext.datastore_admin import backup_handler
  from google.appengine.ext.datastore_admin import config
  from google.appengine.ext.datastore_admin import delete_handler
  from google.appengine.ext.datastore_admin import utils
  
  
  
  
  
  
  ENTITY_ACTIONS = {
      'Delete Entities': delete_handler.ConfirmDeleteHandler.Render,
      'Backup Entities': backup_handler.ConfirmBackupHandler.Render,
  }
  
  
  BACKUP_ACTIONS = {
      'Delete': backup_handler.ConfirmDeleteBackupHandler.Render,
      'Restore': backup_handler.ConfirmRestoreFromBackupHandler.Render,
      'Info': backup_handler.BackupInformationHandler.Render,
  }
  
  PENDING_BACKUP_ACTIONS = {
      'Abort': backup_handler.ConfirmAbortBackupHandler.Render,
      'Info': backup_handler.BackupInformationHandler.Render,
  }
  
  GET_ACTIONS = ENTITY_ACTIONS.copy()
  GET_ACTIONS.update(BACKUP_ACTIONS)
  GET_ACTIONS.update(PENDING_BACKUP_ACTIONS)
  GET_ACTIONS.update({'Import Backup Information':
                      backup_handler.ConfirmBackupImportHandler.Render})
  
  
  def _GetDatastoreStats(kinds_list, use_stats_kinds=False):
    """Retrieves stats for kinds.
  
    Args:
      kinds_list: List of known kinds.
      use_stats_kinds: If stats are available, kinds_list will be ignored and
        all kinds found in stats will be used instead.
  
    Returns:
      timestamp: records time that statistics were last updated.
      kind_dict: dictionary of kind objects with the following members:
      - kind_name: the name of this kind.
      - count: number of known entities of this type.
      - total_bytes_str: total bytes for this kind as a string.
      - average_bytes_str: average bytes per entity as a string.
    """
    global_stat = stats.GlobalStat.all().fetch(1)
    if not global_stat:
      return _KindsListToTuple(kinds_list)
  
    global_ts = global_stat[0].timestamp
  
    kind_stats = stats.KindStat.all().filter('timestamp =', global_ts).fetch(1000)
    if not kind_stats:
      return _KindsListToTuple(kinds_list)
  
    results = {}
    for kind_ent in kind_stats:
  
  
  
      if (not kind_ent.kind_name.startswith('__')
          and (use_stats_kinds or kind_ent.kind_name in kinds_list)
          and kind_ent.count > 0):
        results[kind_ent.kind_name] = _PresentatableKindStats(kind_ent)
  
    utils.CacheStats(results.values())
  
  
  
    for kind_str in kinds_list or []:
      if kind_str not in results:
        results[kind_str] = {'kind_name': kind_str}
  
    return (global_ts,
            sorted(results.values(), key=lambda x: x['kind_name']))
  
  
  def _KindsListToTuple(kinds_list):
    """Build default tuple when no datastore statistics are available. """
    return '', [{'kind_name': kind} for kind in sorted(kinds_list)]
  
  
  def _PresentatableKindStats(kind_ent):
    """Generate dict of presentable values for template."""
    count = kind_ent.count
    entity_bytes = kind_ent.entity_bytes
    total_bytes = kind_ent.bytes
    average_bytes = entity_bytes / count
    return {'kind_name': kind_ent.kind_name,
            'count': utils.FormatThousands(kind_ent.count),
            'entity_bytes_str': utils.GetPrettyBytes(entity_bytes),
            'entity_bytes': entity_bytes,
            'total_bytes_str': utils.GetPrettyBytes(total_bytes),
            'total_bytes': total_bytes,
            'average_bytes_str': utils.GetPrettyBytes(average_bytes),
           }
  
  
  class RouteByActionHandler(webapp.RequestHandler):
    """Route to the appropriate handler based on the action parameter."""
  
    def ListActions(self, error=None):
      """Handler for get requests to datastore_admin/confirm_delete."""
      use_stats_kinds = False
      kinds = []
      more_kinds = False
      try:
        kinds, more_kinds = utils.GetKinds()
        if not kinds:
          use_stats_kinds = True
          logging.warning('Found no kinds. Using datastore stats instead.')
      except datastore_errors.Error, e:
        logging.exception(e)
        use_stats_kinds = True
  
      last_stats_update, kind_stats = _GetDatastoreStats(
          kinds, use_stats_kinds=use_stats_kinds)
  
      template_params = {
          'run_as_a_service': self.request.get('run_as_a_service'),
          'datastore_admin_home': utils.GenerateHomeUrl(None),
          'offer_service': (self.request.get('service') and not
                            self.request.get('run_as_a_service')),
          'kind_stats': kind_stats,
          'more_kinds': more_kinds,
          'last_stats_update': last_stats_update,
          'app_id': self.request.get('app_id'),
  
  
  [... the rest is omitted ...]
files: AppServer/google/appengine/ext/datastore_admin/main.py
files: {% extends "base.html" %}
  {% block title %}Confirm Import of backup information{% endblock %}
  {% block body %}
    <h2>Datastore Admin: Import backup information</h2>
    {% if error %}
      <div class="ae-errorbox">{{ error|escape }}</div>
      <a href="{{ datastore_admin_home }}"><button>Back</button></a>
    {% else %}
      {% if not other_backup_info_files and not selected_backup_info_file %}
        <div class="ae-errorbox">No backup information files were found.</div>
        <a href="{{ datastore_admin_home }}"><button>Back</button></a>
      {% else %}
        <form action="{{base_path}}/{{form_target}}" method="post" style="width:39.39em;">
          <input type="hidden" name="xsrf_token" value="{{xsrf_token|escape}}">
          {% if not other_backup_info_files %}
            <b>Confirm import of:</b>
            <input type="hidden" name="gs_handle" value="{{selected_backup_info_file}}">{{selected_backup_info_file|escape}}
          {% else %}
            {% if selected_backup_info_file %}
              <b>Confirm import of:</b><br/>
              <input type="radio" name="gs_handle" value="{{selected_backup_info_file}}" checked>{{selected_backup_info_file|escape}}</br>
            {% else %}
              {% if backup_info_specified %}
                <b>Requested backup was not found, available backups:</b><br/>
              {% else %}
                <b>Select an available backup:</b><br/>
              {% endif %}
            {% endif %}
            {% for backup_info_file in other_backup_info_files %}
              <input type="radio" name="gs_handle" value="{{backup_info_file}}">{{backup_info_file|escape}}</br>
            {% endfor %}
          {% endif %}
          {% if run_as_a_service %}
            <input type="hidden" name="run_as_a_service" value="{{ run_as_a_service|escape }}">
          {% endif %}
          <input type="hidden" name="service_account_name" value="{{ service_account_name }}">
          <input type="Submit" name="Import" value="Add to backup list">
          <input type="Submit" name="Restore" value="Restore from backup">
          <a href="{{ datastore_admin_home }}">Cancel</a>
        </form>
      {% endif %}
    {% endif %}
  {% endblock %}
  
  
  
  [... the rest is omitted ...]
files: AppServer/google/appengine/ext/datastore_admin/templates/confirm_backup_import.html
files: {% extends "base.html" %}
  {% block title %}Confirm Import of backup information{% endblock %}
  {% block body %}
    <h2>Datastore Admin: Import backup information</h2>
    {% if error %}
      <div class="ae-errorbox">{{ error|escape }}</div>
      <a href="{{ datastore_admin_home }}"><button>Back</button></a>
    {% else %}
      {% if not other_backup_info_files and not selected_backup_info_file %}
        <div class="ae-errorbox">No backup information files were found.</div>
        <a href="{{ datastore_admin_home }}"><button>Back</button></a>
      {% else %}
        <form action="{{base_path}}/{{form_target}}" method="post" style="width:39.39em;">
          <input type="hidden" name="xsrf_token" value="{{xsrf_token|escape}}">
          {% if not other_backup_info_files %}
            <b>Confirm import of:</b>
            <input type="hidden" name="gs_handle" value="{{selected_backup_info_file}}">{{selected_backup_info_file|escape}}
          {% else %}
            {% if selected_backup_info_file %}
              <b>Confirm import of:</b><br/>
              <input type="radio" name="gs_handle" value="{{selected_backup_info_file}}" checked>{{selected_backup_info_file|escape}}</br>
            {% else %}
              {% if backup_info_specified %}
                <b>Requested backup was not found, available backups:</b><br/>
              {% else %}
                <b>Select an available backup:</b><br/>
              {% endif %}
            {% endif %}
            {% for backup_info_file in other_backup_info_files %}
              <input type="radio" name="gs_handle" value="{{backup_info_file}}">{{backup_info_file|escape}}</br>
            {% endfor %}
          {% endif %}
          {% if run_as_a_service %}
            <input type="hidden" name="run_as_a_service" value="{{ run_as_a_service|escape }}">
          {% endif %}
          <input type="Submit" name="Import" value="Add to backup list">
          <input type="Submit" name="Restore" value="Restore from backup">
          <a href="{{ datastore_admin_home }}">Cancel</a>
        </form>
      {% endif %}
    {% endif %}
  {% endblock %}
  
  
  
  [... the rest is omitted ...]
files: AppServer/google/appengine/ext/datastore_admin/templates/confirm_backup_import.html
files: {% extends "base.html" %}
  {% block title %}Confirm Restore from a backup copy{% endblock %}
  {% block body %}
    <h2>Datastore Admin: Restore from a backup copy</h2>
    {% if backup %}
      Are you sure you want to restore from {{ backup.name }}?
      <br/>Restore does not delete any new entities added since the backup.
      However, it will replace existing entities with the backed-up version for the following kinds:
      <form action="{{base_path}}/{{form_target}}" method="post" style="width:39.39em;">
        {% if run_as_a_service %}
          <input type="hidden" name="run_as_a_service" value="{{ run_as_a_service|escape }}">
        {% endif %}
        <input type="hidden" name="backup_id" value="{{backup.key}}">
        <input type="hidden" name="xsrf_token" value="{{xsrf_token|escape}}">
        <input type="hidden" name="delete_backup_after_restore" value="{{ delete_backup_after_restore|escape }}">
        <ul>
          {% for kind in backup.kinds %}
            <li><input type="checkbox" name="kind" value="{{kind}}" checked="checked"/>{{ kind }}</li>
          {% endfor %}
        </ul>
        {% if notreadonly_warning %}
        <div>
          <h4 style="padding:0;margin-top:1em;margin-bottom:0.46em;">WARNING</h4>
          {% if original_app_warning %}
            <ul><li>
          {% else %}
            <p>
          {% endif %}
          The application is not in read-only mode and restore may
          conflict with other changes.
          To place the application in read-only mode,
          please follow the instructions at
          <a href="https://developers.google.com/appengine/docs/adminconsole/applicationsettings#Disable_Datastore_Writes" target="_blank">
          Disable Datastore Writes</a>
          {% if not original_app_warning %}
            </p>
            </div>
          {% endif %}
        {% endif %}
        {% if original_app_warning %}
          {% if not notreadonly_warning %}
            <div>
            <h4 style="padding:0;margin-top:1em;margin-bottom:0.46em;">WARNING</h4>
            <p>
          {% else %}
            <li>
          {% endif %}
          The backup data comes from a different application, {{original_app_warning|escape}}.
          {% if not notreadonly_warning %}
            </p>
          {% else %}
            </li>
          {% endif %}
          </div>
        {% endif %}
        <p>Operation will be performed using the selected queue:
          {% if queues %}
          <select name="queue">
            {% for queue in queues %}
              <option value="{{ queue }}">{{ queue }}</option>
            {% endfor %}
          </select>
          {% else %}
          <input type="text" name="queue" value="default"/>
          {% endif %}
        </p>
        <p>
          <label for="service_account_name">
            Service account name
            <img class="ae-help-icon" src="{{ base_path }}/static/img/help.gif" height="14" width="14" alt="help"
                 title="You can define additional service accounts using the dashboard.">
          </label>
          <select id="service_account_name" name="service_account_name">
            <option value="">--Choose an option--</option>
            {% for account_name in service_accounts %}
            <option value="{{ account_name }}">{{ account_name }}</option>
            {% endfor %}
          </select>
        </p>
        <input type="submit" value="Restore">
        <a href="{{ datastore_admin_home }}">Cancel</a>
      </form>
    {% else %}
      <div class="ae-errorbox">Backup was not found.</div>
    {% endif %}
  {% endblock %}
  
  
  [... the rest is omitted ...]
files: AppServer/google/appengine/ext/datastore_admin/templates/confirm_restore_from_backup.html
files: {% extends "base.html" %}
  {% block title %}Confirm Restore from a backup copy{% endblock %}
  {% block body %}
    <h2>Datastore Admin: Restore from a backup copy</h2>
    {% if backup %}
      Are you sure you want to restore from {{ backup.name }}?
      <br/>Restore does not delete any new entities added since the backup.
      However, it will replace existing entities with the backed-up version for the following kinds:
      <form action="{{base_path}}/{{form_target}}" method="post" style="width:39.39em;">
        {% if run_as_a_service %}
          <input type="hidden" name="run_as_a_service" value="{{ run_as_a_service|escape }}">
        {% endif %}
        <input type="hidden" name="backup_id" value="{{backup.key}}">
        <input type="hidden" name="xsrf_token" value="{{xsrf_token|escape}}">
        <input type="hidden" name="delete_backup_after_restore" value="{{ delete_backup_after_restore|escape }}">
        <ul>
          {% for kind in backup.kinds %}
            <li><input type="checkbox" name="kind" value="{{kind}}" checked="checked"/>{{ kind }}</li>
          {% endfor %}
        </ul>
        {% if notreadonly_warning %}
        <div>
          <h4 style="padding:0;margin-top:1em;margin-bottom:0.46em;">WARNING</h4>
          {% if original_app_warning %}
            <ul><li>
          {% else %}
            <p>
          {% endif %}
          The application is not in read-only mode and restore may
          conflict with other changes.
          To place the application in read-only mode,
          please follow the instructions at
          <a href="https://developers.google.com/appengine/docs/adminconsole/applicationsettings#Disable_Datastore_Writes" target="_blank">
          Disable Datastore Writes</a>
          {% if not original_app_warning %}
            </p>
            </div>
          {% endif %}
        {% endif %}
        {% if original_app_warning %}
          {% if not notreadonly_warning %}
            <div>
            <h4 style="padding:0;margin-top:1em;margin-bottom:0.46em;">WARNING</h4>
            <p>
          {% else %}
            <li>
          {% endif %}
          The backup data comes from a different application, {{original_app_warning|escape}}.
          {% if not notreadonly_warning %}
            </p>
          {% else %}
            </li>
          {% endif %}
          </div>
        {% endif %}
        <p>Operation will be performed using the selected queue:
          {% if queues %}
          <select name="queue">
            {% for queue in queues %}
              <option value="{{ queue }}">{{ queue }}</option>
            {% endfor %}
          </select>
          {% else %}
          <input type="text" name="queue" value="default"/>
          {% endif %}
        </p>
        <input type="submit" value="Restore">
        <a href="{{ datastore_admin_home }}">Cancel</a>
      </form>
    {% else %}
      <div class="ae-errorbox">Backup was not found.</div>
    {% endif %}
  {% endblock %}
  
  
  [... the rest is omitted ...]
files: AppServer/google/appengine/ext/datastore_admin/templates/confirm_restore_from_backup.html
files: {% extends "base.html" %}
  {% block body %}
    <h2>Datastore Admin of {{hosting_app_id}}</h2>
    {% if run_as_a_service or offer_service %}
      <div class="ae-promo ae-rounded-sml">
        <strong>
          The managed backup service is deprecated, and will no longer be available as of
          Mar 27th, 2018.
        </strong>
      </div>
    {% endif %}
    {% if run_as_a_service %}
      <div class="ae-message ae-rounded-sml">
        Backup/Restore running as a service.
        <a href="{{datastore_admin_home}}?service=T">Disable</a>
      </div>
    {% else %}
      {% if offer_service %}
        <div class="ae-message ae-rounded-sml">
          This application may run backup/restore as a service.
          <a href="{{datastore_admin_home}}?run_as_a_service=True">Enable</a>
        </div>
      {% endif %}
    {% endif %}
    {% if error %}
    <div class="ae-errorbox">
      {{ error|escape }}
    </div>
    {% endif %}
    <div class="ae-table-caption">
      <div class="ae-paginate">
        <div class="g-section g-tpl-50-50 g-split">
          <div class="g-unit g-first">
            <strong>Entities</strong>
          </div>
          <div class="g-unit">
            <span id="ae-dash-quota-refresh-info">
            {% if last_stats_update %}
              Entity statistics last updated
              {{ last_stats_update|date:"M j, Y g:i a T" }}
            {% else %}
              Entity statistics are temporarily unavailable and should appear within 48 hours.
            {% endif %}
            <a target="_blank"
               href="https://cloud.google.com/appengine/docs/standard/python/console/datastore-statistics">
              <img class="ae-help-icon" src="{{ base_path }}/static/img/help.gif"
                height="14" width="14" alt="help">
            </a>
            </span>
          </div>
        </div>
      </div>
    </div>
    {% if kind_stats %}
    <form action="{{ base_path }}/" method="post">
      {% if run_as_a_service %}
        <input type="hidden" name="run_as_a_service" value="{{ run_as_a_service|escape }}">
      {% endif %}
      {% if has_namespace %}
      <input type="hidden" name="namespace" value="{{ namespace|escape }}">
      {% endif %}
      <input type="hidden" name="app_id" value="{{ app_id|escape }}">
      <table id="kinds" class="ae-table ae-table-striped">
        <thead>
          <tr>
            <th scope="col" class="cbc tct-selectall"></th>
            <th scope="col" class="id">Entity Kind</th>
            <th scope="col" class="id"># Entities</th>
            <th scope="col" class="id">Avg. Size/Entity</th>
            <th scope="col" class="id">Entities Size</th>
            <th scope="col" class="id">Total Size</th>
          </tr>
        </thead>
        <tbody>
          {% for kind in kind_stats %}
            <tr>
              <td class="cbc">
                <input type="checkbox" name="kind" value="{{ kind.kind_name }}"
                       id="{{ kind.kind_name }}">
              </td>
              <td class="id">
                <label for="{{ kind.kind_name }}">{{ kind.kind_name }}</label>
              </td>
              {% if kind.count %}
                <td class="id">{{ kind.count }}</td>
                <td class="id">{{ kind.average_bytes_str }}</td>
                <td class="id">{{ kind.entity_bytes_str }}</td>
                <td class="id">{{ kind.total_bytes_str }}</td>
              {% else %}
                <td class="id" style="color: #666667;">Stats not available</td>
                <td class="id" style="color: #666667;">Stats not available</td>
                <td class="id" style="color: #666667;">Stats not available</td>
                <td class="id" style="color: #666667;">Stats not available</td>
              {% endif %}
            </tr>
          {% endfor %}
          {% if more_kinds %}
            <tr>
              <td colspan="6" class="cbc">
                Not all kinds could be displayed, possibly because there were too many. To back up a kind that does not appear in this list, use a <a href="https://developers.google.com/appengine/articles/scheduled_backups" target="_blank">scheduled backup</a>.
              </td>
            </tr>
          {% endif %}
        </tbody>
        <tfoot>
          <tr>
            <td colspan="6">
              {% for action in action_list %}
                <input type="submit" name="action"
                  class="goog-button" value="{{ action }}">
              {% endfor %}
            </td>
          </tr>
        </tfoot>
      </table>
    </form>
    {% else %}
      <table class="ae-table">
        <tbody>
          <tr>
            <td>
              <div class="ae-disabled"
                  style="padding-top: 0.75em; padding-bottom: 0.75em;
                         text-align: center;">
                Datastore is empty.
              </div>
            </td>
          </tr>
        </tbody>
      </table>
    {% endif %}
    {% if pending_backups %}
      <p>
      <div class="ae-table-caption">
        <div class="ae-paginate">
          <div class="g-section g-tpl-50-50 g-split">
            <div class="g-unit g-first">
              <strong>Pending Backups</strong>
            </div>
          </div>
        </div>
      </div>
      <form action="{{ base_path }}/" method="post">
        {% if run_as_a_service %}
          <input type="hidden
  
  [... the rest is omitted ...]
files: AppServer/google/appengine/ext/datastore_admin/templates/list_actions.html
files: {% extends "base.html" %}
  {% block body %}
    <h2>Datastore Admin of {{hosting_app_id}}</h2>
    {% if run_as_a_service or offer_service %}
      <div class="ae-promo ae-rounded-sml">
        <strong>
          The managed backup service is deprecated, and will no longer be available as of
          Mar 27th, 2018.
        </strong>
      </div>
    {% endif %}
    {% if run_as_a_service %}
      <div class="ae-message ae-rounded-sml">
        Backup/Restore running as a service.
        <a href="{{datastore_admin_home}}?service=T">Disable</a>
      </div>
    {% else %}
      {% if offer_service %}
        <div class="ae-message ae-rounded-sml">
          This application may run backup/restore as a service.
          <a href="{{datastore_admin_home}}?run_as_a_service=True">Enable</a>
        </div>
      {% endif %}
    {% endif %}
    {% if error %}
    <div class="ae-errorbox">
      {{ error|escape }}
    </div>
    {% endif %}
    <div class="ae-table-caption">
      <div class="ae-paginate">
        <div class="g-section g-tpl-50-50 g-split">
          <div class="g-unit g-first">
            <strong>Entities</strong>
          </div>
          <div class="g-unit">
            <span id="ae-dash-quota-refresh-info">
            {% if last_stats_update %}
              Entity statistics last updated
              {{ last_stats_update|date:"M j, Y g:i a T" }}
            {% else %}
              Entity statistics are temporarily unavailable and should appear within 48 hours.
            {% endif %}
            <a target="_blank"
               href="https://cloud.google.com/appengine/docs/standard/python/console/datastore-statistics">
              <img class="ae-help-icon" src="{{ base_path }}/static/img/help.gif"
                height="14" width="14" alt="help">
            </a>
            </span>
          </div>
        </div>
      </div>
    </div>
    {% if kind_stats %}
    <form action="{{ base_path }}/" method="post">
      {% if run_as_a_service %}
        <input type="hidden" name="run_as_a_service" value="{{ run_as_a_service|escape }}">
      {% endif %}
      {% if has_namespace %}
      <input type="hidden" name="namespace" value="{{ namespace|escape }}">
      {% endif %}
      <input type="hidden" name="app_id" value="{{ app_id|escape }}">
      <table id="kinds" class="ae-table ae-table-striped">
        <thead>
          <tr>
            <th scope="col" class="cbc tct-selectall"></th>
            <th scope="col" class="id">Entity Kind</th>
            <th scope="col" class="id"># Entities</th>
            <th scope="col" class="id">Avg. Size/Entity</th>
            <th scope="col" class="id">Entities Size</th>
            <th scope="col" class="id">Total Size</th>
          </tr>
        </thead>
        <tbody>
          {% for kind in kind_stats %}
            <tr>
              <td class="cbc">
                <input type="checkbox" name="kind" value="{{ kind.kind_name }}"
                       id="{{ kind.kind_name }}">
              </td>
              <td class="id">
                <label for="{{ kind.kind_name }}">{{ kind.kind_name }}</label>
              </td>
              {% if kind.count %}
                <td class="id">{{ kind.count }}</td>
                <td class="id">{{ kind.average_bytes_str }}</td>
                <td class="id">{{ kind.entity_bytes_str }}</td>
                <td class="id">{{ kind.total_bytes_str }}</td>
              {% else %}
                <td class="id" style="color: #666667;">Stats not available</td>
                <td class="id" style="color: #666667;">Stats not available</td>
                <td class="id" style="color: #666667;">Stats not available</td>
                <td class="id" style="color: #666667;">Stats not available</td>
              {% endif %}
            </tr>
          {% endfor %}
          {% if more_kinds %}
            <tr>
              <td colspan="6" class="cbc">
                Not all kinds could be displayed, possibly because there were too many. To back up a kind that does not appear in this list, use a <a href="https://developers.google.com/appengine/articles/scheduled_backups" target="_blank">scheduled backup</a>.
              </td>
            </tr>
          {% endif %}
        </tbody>
        <tfoot>
          <tr>
            <td colspan="6">
              {% for action in action_list %}
                <input type="submit" name="action"
                  class="goog-button" value="{{ action }}">
              {% endfor %}
            </td>
          </tr>
        </tfoot>
      </table>
    </form>
    {% else %}
      <table class="ae-table">
        <tbody>
          <tr>
            <td>
              <div class="ae-disabled"
                  style="padding-top: 0.75em; padding-bottom: 0.75em;
                         text-align: center;">
                Datastore is empty.
              </div>
            </td>
          </tr>
        </tbody>
      </table>
    {% endif %}
    {% if pending_backups %}
      <p>
      <div class="ae-table-caption">
        <div class="ae-paginate">
          <div class="g-section g-tpl-50-50 g-split">
            <div class="g-unit g-first">
              <strong>Pending Backups</strong>
            </div>
          </div>
        </div>
      </div>
      <form action="{{ base_path }}/" method="post">
        {% if run_as_a_service %}
          <input type="hidden
  
  [... the rest is omitted ...]
files: AppServer/google/appengine/ext/datastore_admin/templates/list_actions.html
files: #!/usr/bin/env python
  #
  # Copyright 2007 Google Inc.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  #
  
  
  
  
  """Used render templates for datastore admin."""
  
  
  
  import base64
  import datetime
  import json
  import logging
  import os
  import random
  import time
  
  # AppScale: Use bundled mapreduce library.
  from google.appengine.ext.mapreduce import context
  from google.appengine.ext.mapreduce import control
  from google.appengine.ext.mapreduce import model
  from google.appengine.ext.mapreduce import operation as mr_operation
  from google.appengine.ext.mapreduce import util
  
  from google.appengine.api import app_identity
  from google.appengine.api import datastore
  from google.appengine.api import datastore_errors
  from google.appengine.api import memcache
  from google.appengine.api import urlfetch
  from google.appengine.api import users
  from google.appengine.datastore import datastore_rpc
  from google.appengine.datastore import entity_pb
  from google.appengine.ext import db
  import webapp2 as webapp
  from google.appengine.ext.datastore_admin import config
  from google.appengine.ext.db import metadata
  from google.appengine.ext.db import stats
  from google.appengine.ext.webapp import _template
  from google.appengine.runtime import apiproxy_errors
  
  
  MEMCACHE_NAMESPACE = '_ah-datastore_admin'
  XSRF_VALIDITY_TIME = 600
  KINDS_AND_SIZES_VAR = 'kinds_and_sizes'
  MAPREDUCE_MIN_SHARDS = 8
  MAPREDUCE_DEFAULT_SHARDS = 32
  MAPREDUCE_MAX_SHARDS = 256
  RESERVE_KEY_POOL_MAX_SIZE = 1000
  
  DATASTORE_ADMIN_OPERATION_KIND = '_AE_DatastoreAdmin_Operation'
  BACKUP_INFORMATION_KIND = '_AE_Backup_Information'
  BACKUP_INFORMATION_FILES_KIND = '_AE_Backup_Information_Kind_Files'
  BACKUP_INFORMATION_KIND_TYPE_INFO = '_AE_Backup_Information_Kind_Type_Info'
  DATASTORE_ADMIN_KINDS = (DATASTORE_ADMIN_OPERATION_KIND,
                           BACKUP_INFORMATION_KIND,
                           BACKUP_INFORMATION_FILES_KIND,
                           BACKUP_INFORMATION_KIND_TYPE_INFO)
  
  
  MAX_RPCS = 10
  
  
  def IsKindNameVisible(kind_name):
    return not (kind_name.startswith('__') or
                kind_name in DATASTORE_ADMIN_KINDS or
                kind_name in model._MAP_REDUCE_KINDS)
  
  
  def RenderToResponse(handler, template_file, template_params):
    """Render the given template_file using template_vals and write to response.
  
    Args:
      handler: the handler whose response we should render to
      template_file: the file name only of the template file we are using
      template_params: the parameters used to render the given template
    """
    template_params = _GetDefaultParams(template_params)
  
  
  
  
  
  
  
  
    handler.response.headers['X-FRAME-OPTIONS'] = ('ALLOW-FROM %s' %
                                                   config.ADMIN_CONSOLE_URL)
    template_params['admin_console_url'] = config.ADMIN_CONSOLE_URL
  
    rendered = _template.render(_GetTemplatePath(template_file), template_params)
    handler.response.out.write(rendered)
  
  
  def _GetTemplatePath(template_file):
    """Return the expected path for the template to render.
  
    Args:
      template_file: simple file name of template to render.
  
    Returns:
      path of template to render.
    """
    return os.path.join(
        os.path.dirname(__file__), 'templates', template_file)
  
  
  def _GetDefaultParams(template_params):
    """Update template_params to always contain necessary paths and never None."""
    if not template_params:
      template_params = {}
    template_params.update({
        'base_path': config.BASE_PATH,
        'mapreduce_path': config.MAPREDUCE_PATH,
    })
    return template_params
  
  
  def CreateXsrfToken(action):
    """Generate a token to be passed with a form for XSRF protection.
  
    Args:
      action: action to restrict token to
  
    Returns:
      suitably random token which is only valid for ten minutes and, if the user
      is authenticated, is only valid for the user that generated it.
    """
    user_str = _MakeUserStr()
  
    token = base64.b64encode(
        ''.join(chr(int(random.random()*255)) for _ in range(0, 64)))
  
    memcache.set(token,
                 (user_str, action),
                 time=XSRF_VALIDITY_TIME,
                 namespace=MEMCACHE_NAMESPACE)
  
    return token
  
  
  def ValidateXsrfToken(token, action):
    """Validate a given XSRF token by retrieving it from memcache.
  
    If the token has not been evicted from memcache (past ten minutes) and the
    user strings are equal, then this is a valid token.
  
    Args:
      token: token to validate from memcache.
      action: action that token should correspond to
  
    Returns:
      True if the token exists in memcache and the user strings are equal,
      False otherwise.
    """
    user_str = _MakeUserStr()
    token_obj = memcache.get(token, namespace=MEMCACHE_NAMESPACE)
  
    if not token_obj:
      return False
  
    token_str, token_action = token_obj
    if user_str != token_str or action != token_action:
      return False
  
    return True
  
  
  def CacheStats(formatted_results):
    """Cache last retrieved kind size values in memcache.
  
    Args:
      formatted_results: list of dictionaries
  
  [... the rest is omitted ...]
files: AppServer/google/appengine/ext/datastore_admin/utils.py
files: #!/usr/bin/env python
  #
  # Copyright 2007 Google Inc.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  #
  
  
  
  
  """Used render templates for datastore admin."""
  
  
  
  import base64
  import datetime
  import logging
  import os
  import random
  import time
  
  # AppScale: Use bundled mapreduce library.
  from google.appengine.ext.mapreduce import context
  from google.appengine.ext.mapreduce import control
  from google.appengine.ext.mapreduce import model
  from google.appengine.ext.mapreduce import operation as mr_operation
  from google.appengine.ext.mapreduce import util
  
  from google.appengine.api import datastore
  from google.appengine.api import datastore_errors
  from google.appengine.api import memcache
  from google.appengine.api import users
  from google.appengine.datastore import datastore_rpc
  from google.appengine.datastore import entity_pb
  from google.appengine.ext import db
  import webapp2 as webapp
  from google.appengine.ext.datastore_admin import config
  from google.appengine.ext.db import metadata
  from google.appengine.ext.db import stats
  from google.appengine.ext.webapp import _template
  from google.appengine.runtime import apiproxy_errors
  
  
  MEMCACHE_NAMESPACE = '_ah-datastore_admin'
  XSRF_VALIDITY_TIME = 600
  KINDS_AND_SIZES_VAR = 'kinds_and_sizes'
  MAPREDUCE_MIN_SHARDS = 8
  MAPREDUCE_DEFAULT_SHARDS = 32
  MAPREDUCE_MAX_SHARDS = 256
  RESERVE_KEY_POOL_MAX_SIZE = 1000
  
  DATASTORE_ADMIN_OPERATION_KIND = '_AE_DatastoreAdmin_Operation'
  BACKUP_INFORMATION_KIND = '_AE_Backup_Information'
  BACKUP_INFORMATION_FILES_KIND = '_AE_Backup_Information_Kind_Files'
  BACKUP_INFORMATION_KIND_TYPE_INFO = '_AE_Backup_Information_Kind_Type_Info'
  DATASTORE_ADMIN_KINDS = (DATASTORE_ADMIN_OPERATION_KIND,
                           BACKUP_INFORMATION_KIND,
                           BACKUP_INFORMATION_FILES_KIND,
                           BACKUP_INFORMATION_KIND_TYPE_INFO)
  
  
  MAX_RPCS = 10
  
  
  def IsKindNameVisible(kind_name):
    return not (kind_name.startswith('__') or
                kind_name in DATASTORE_ADMIN_KINDS or
                kind_name in model._MAP_REDUCE_KINDS)
  
  
  def RenderToResponse(handler, template_file, template_params):
    """Render the given template_file using template_vals and write to response.
  
    Args:
      handler: the handler whose response we should render to
      template_file: the file name only of the template file we are using
      template_params: the parameters used to render the given template
    """
    template_params = _GetDefaultParams(template_params)
  
  
  
  
  
  
  
  
    handler.response.headers['X-FRAME-OPTIONS'] = ('ALLOW-FROM %s' %
                                                   config.ADMIN_CONSOLE_URL)
    template_params['admin_console_url'] = config.ADMIN_CONSOLE_URL
  
    rendered = _template.render(_GetTemplatePath(template_file), template_params)
    handler.response.out.write(rendered)
  
  
  def _GetTemplatePath(template_file):
    """Return the expected path for the template to render.
  
    Args:
      template_file: simple file name of template to render.
  
    Returns:
      path of template to render.
    """
    return os.path.join(
        os.path.dirname(__file__), 'templates', template_file)
  
  
  def _GetDefaultParams(template_params):
    """Update template_params to always contain necessary paths and never None."""
    if not template_params:
      template_params = {}
    template_params.update({
        'base_path': config.BASE_PATH,
        'mapreduce_path': config.MAPREDUCE_PATH,
    })
    return template_params
  
  
  def CreateXsrfToken(action):
    """Generate a token to be passed with a form for XSRF protection.
  
    Args:
      action: action to restrict token to
  
    Returns:
      suitably random token which is only valid for ten minutes and, if the user
      is authenticated, is only valid for the user that generated it.
    """
    user_str = _MakeUserStr()
  
    token = base64.b64encode(
        ''.join(chr(int(random.random()*255)) for _ in range(0, 64)))
  
    memcache.set(token,
                 (user_str, action),
                 time=XSRF_VALIDITY_TIME,
                 namespace=MEMCACHE_NAMESPACE)
  
    return token
  
  
  def ValidateXsrfToken(token, action):
    """Validate a given XSRF token by retrieving it from memcache.
  
    If the token has not been evicted from memcache (past ten minutes) and the
    user strings are equal, then this is a valid token.
  
    Args:
      token: token to validate from memcache.
      action: action that token should correspond to
  
    Returns:
      True if the token exists in memcache and the user strings are equal,
      False otherwise.
    """
    user_str = _MakeUserStr()
    token_obj = memcache.get(token, namespace=MEMCACHE_NAMESPACE)
  
    if not token_obj:
      return False
  
    token_str, token_action = token_obj
    if user_str != token_str or action != token_action:
      return False
  
    return True
  
  
  def CacheStats(formatted_results):
    """Cache last retrieved kind size values in memcache.
  
    Args:
      formatted_results: list of dictionaries of the form returnned by
        main._PresentableKindStats.
    """
    kinds_and_sizes
  
  [... the rest is omitted ...]
files: AppServer/google/appengine/ext/datastore_admin/utils.py