files.new_contents: #!/usr/bin/env python
  #
  # Copyright 2007 Google Inc.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  #
  
  
  
  
  
  """Handler for data backup operation.
  
  Generic datastore admin console transfers control to ConfirmBackupHandler
  after selection of entities. The ConfirmBackupHandler confirms with user
  his choice, enters a backup name and transfers control to
  DoBackupHandler. DoBackupHandler starts backup mappers and displays confirmation
  page.
  
  This module also contains actual mapper code for backing data over.
  """
  
  
  
  
  
  
  
  
  from __future__ import with_statement
  
  
  
  import copy
  import cStringIO
  import datetime
  import itertools
  import logging
  import os
  import random
  import re
  import time
  import urllib
  import xml.dom.minidom
  
  from google.appengine._internal import cloudstorage
  
  # AppScale: Use bundled mapreduce library.
  from google.appengine.ext.mapreduce import context
  from google.appengine.ext.mapreduce import datastore_range_iterators as db_iters
  from google.appengine.ext.mapreduce import input_readers
  from google.appengine.ext.mapreduce import json_util
  from google.appengine.ext.mapreduce import operation as op
  from google.appengine.ext.mapreduce import output_writers
  from google.appengine.ext.mapreduce import parameters
  from google.appengine.ext.mapreduce import records
  
  from google.appengine.api import apiproxy_stub_map
  from google.appengine.api import app_identity
  from google.appengine.api import blobstore as blobstore_api
  from google.appengine.api import capabilities
  from google.appengine.api import datastore
  from google.appengine.api import datastore_errors
  from google.appengine.api import datastore_types
  from google.appengine.api import taskqueue
  from google.appengine.api import urlfetch
  from google.appengine.api.taskqueue import taskqueue_service_pb
  from google.appengine.datastore import entity_pb
  from google.appengine.ext import blobstore
  from google.appengine.ext import db
  from google.appengine.ext import deferred
  import webapp2 as webapp
  from google.appengine.runtime import apiproxy_errors
  from google.appengine.ext.datastore_admin import backup_pb2
  from google.appengine.ext.datastore_admin import config
  from google.appengine.ext.datastore_admin import utils
  
  
  XSRF_ACTION = 'backup'
  BUCKET_PATTERN = (r'^([a-zA-Z0-9]+([\-_]+[a-zA-Z0-9]+)*)'
                    r'(\.([a-zA-Z0-9]+([\-_]+[a-zA-Z0-9]+)*))*$')
  MAX_BUCKET_LEN = 222
  MIN_BUCKET_LEN = 3
  MAX_BUCKET_SEGMENT_LEN = 63
  NUM_KINDS_DEFERRED_THRESHOLD = 10
  MAX_BLOBS_PER_DELETE = 500
  TEST_WRITE_FILENAME_PREFIX = 'datastore_backup_write_test'
  MAX_KEYS_LIST_SIZE = 100
  MAX_TEST_FILENAME_TRIES = 10
  
  MEANING_TO_PRIMITIVE_TYPE = {
      entity_pb.Property.GD_WHEN: backup_pb2.EntitySchema.DATE_TIME,
      entity_pb.Property.GD_RATING: backup_pb2.EntitySchema.RATING,
      entity_pb.Property.ATOM_LINK: backup_pb2.EntitySchema.LINK,
      entity_pb.Property.ATOM_CATEGORY: backup_pb2.EntitySchema.CATEGORY,
      entity_pb.Property.GD_PHONENUMBER: backup_pb2.EntitySchema.PHONE_NUMBER,
      entity_pb.Property.GD_POSTALADDRESS: backup_pb2.EntitySchema.POSTAL_ADDRESS,
      entity_pb.Property.GD_EMAIL: backup_pb2.EntitySchema.EMAIL,
      entity_pb.Property.GD_IM: backup_pb2.EntitySchema.IM_HANDLE,
      entity_pb.Property.BLOBKEY: backup_pb2.EntitySchema.BLOB_KEY,
      entity_pb.Property.TEXT: backup_pb2.EntitySchema.TEXT,
      entity_pb.Property.BLOB: backup_pb2.EntitySchema.BLOB,
  
  
  [... the rest is omitted ...]
files.new_path: AppServer/google/appengine/ext/datastore_admin/backup_handler.py
files.old_contents: #!/usr/bin/env python
  #
  # Copyright 2007 Google Inc.
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  #     http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  #
  
  
  
  
  
  """Handler for data backup operation.
  
  Generic datastore admin console transfers control to ConfirmBackupHandler
  after selection of entities. The ConfirmBackupHandler confirms with user
  his choice, enters a backup name and transfers control to
  DoBackupHandler. DoBackupHandler starts backup mappers and displays confirmation
  page.
  
  This module also contains actual mapper code for backing data over.
  """
  
  
  
  
  
  
  
  
  from __future__ import with_statement
  
  
  
  import copy
  import cStringIO
  import datetime
  import itertools
  import logging
  import os
  import random
  import re
  import time
  import urllib
  import xml.dom.minidom
  
  from google.appengine._internal import cloudstorage
  
  # AppScale: Use bundled mapreduce library.
  from google.appengine.ext.mapreduce import context
  from google.appengine.ext.mapreduce import datastore_range_iterators as db_iters
  from google.appengine.ext.mapreduce import input_readers
  from google.appengine.ext.mapreduce import json_util
  from google.appengine.ext.mapreduce import operation as op
  from google.appengine.ext.mapreduce import output_writers
  from google.appengine.ext.mapreduce import parameters
  from google.appengine.ext.mapreduce import records
  
  from google.appengine.api import apiproxy_stub_map
  from google.appengine.api import app_identity
  from google.appengine.api import blobstore as blobstore_api
  from google.appengine.api import capabilities
  from google.appengine.api import datastore
  from google.appengine.api import datastore_errors
  from google.appengine.api import datastore_types
  from google.appengine.api import taskqueue
  from google.appengine.api import urlfetch
  from google.appengine.api.taskqueue import taskqueue_service_pb
  from google.appengine.datastore import entity_pb
  from google.appengine.ext import blobstore
  from google.appengine.ext import db
  from google.appengine.ext import deferred
  import webapp2 as webapp
  from google.appengine.runtime import apiproxy_errors
  from google.appengine.ext.datastore_admin import backup_pb2
  from google.appengine.ext.datastore_admin import config
  from google.appengine.ext.datastore_admin import utils
  
  
  XSRF_ACTION = 'backup'
  BUCKET_PATTERN = (r'^([a-zA-Z0-9]+([\-_]+[a-zA-Z0-9]+)*)'
                    r'(\.([a-zA-Z0-9]+([\-_]+[a-zA-Z0-9]+)*))*$')
  MAX_BUCKET_LEN = 222
  MIN_BUCKET_LEN = 3
  MAX_BUCKET_SEGMENT_LEN = 63
  NUM_KINDS_DEFERRED_THRESHOLD = 10
  MAX_BLOBS_PER_DELETE = 500
  TEST_WRITE_FILENAME_PREFIX = 'datastore_backup_write_test'
  MAX_KEYS_LIST_SIZE = 100
  MAX_TEST_FILENAME_TRIES = 10
  
  MEANING_TO_PRIMITIVE_TYPE = {
      entity_pb.Property.GD_WHEN: backup_pb2.EntitySchema.DATE_TIME,
      entity_pb.Property.GD_RATING: backup_pb2.EntitySchema.RATING,
      entity_pb.Property.ATOM_LINK: backup_pb2.EntitySchema.LINK,
      entity_pb.Property.ATOM_CATEGORY: backup_pb2.EntitySchema.CATEGORY,
      entity_pb.Property.GD_PHONENUMBER: backup_pb2.EntitySchema.PHONE_NUMBER,
      entity_pb.Property.GD_POSTALADDRESS: backup_pb2.EntitySchema.POSTAL_ADDRESS,
      entity_pb.Property.GD_EMAIL: backup_pb2.EntitySchema.EMAIL,
      entity_pb.Property.GD_IM: backup_pb2.EntitySchema.IM_HANDLE,
      entity_pb.Property.BLOBKEY: backup_pb2.EntitySchema.BLOB_KEY,
      entity_pb.Property.TEXT: backup_pb2.EntitySchema.TEXT,
      entity_pb.Property.BLOB: backup_pb2.EntitySchema.BLOB,
  
  
  [... the rest is omitted ...]
files.old_path: AppServer/google/appengine/ext/datastore_admin/backup_handler.py
files.new_contents: {% extends "base.html" %}
  {% block title %}Confirm Backup of {{kind_str|escape}}{% endblock %}
  {% block body %}
    <h2>Datastore Admin: Backup to Google Cloud Storage</h2>
    {% if kind_list %}
      {% comment %}
      size_total represents the total size of the figures for which we have
      datastore statistics available.
      sizes_known indicates whether or not we have datastore statistics for all
      kinds.
      These two variables are used to render different messages for the three
      possible scenarios: all sizes known, some sizes known, no sizes known.
      {% endcomment %}
  
      <form class="ae-form" action="{{base_path}}/{{form_target}}" method="post"
          style="width:39.39em;">
        {% if run_as_a_service %}
          <input type="hidden" name="run_as_a_service" value="{{ run_as_a_service|escape }}">
        {% endif %}
        {% for kind in kind_list %}
        <input type="hidden" name="kind" value="{{kind|escape}}">
        {% endfor %}
  
        <div class="ae-input-row">
          <div class="ae-label-row">
            <label for="backup_name">Backup name: </label>
          </div>
          <div class="ae-input-row">
            <input type="text" id="backup_name" name="backup_name"
                style="width:100%"
                value="{{backup_name}}">
          </div>
        </div>
        <input type="hidden" name="xsrf_token" value="{{xsrf_token|escape}}">
  
        {% if notreadonly_warning %}
        <div>
          <h4 style="padding:0;margin-top:1em;margin-bottom:0.46em;">WARNING</h4>
          <p>The application is not in read-only mode and updates
          during the backup process may not be captured.
          To place the application in read-only mode,
          please follow the instructions at
          <a href="https://cloud.google.com/datastore/docs/console/managing-datastore#enabling_datastore_writes" target="_blank">
          Enabling or Disabling Datastore Writes</a>
          </p>
        </div>
        {% endif %}
  
        {% if blob_warning %}
        <p>The application has blob data.  BlobInfo records and data will NOT be
        backed up.</p>
        {% endif %}
  
        <p>Selected namespace:
          <select name="namespace">
            {% for namespace in namespaces %}
              <option value="{{ namespace.1 }}" {% if namespace.2 %}selected="selected"{% endif %}>{{ namespace.0 }}</option>
            {% endfor %}
          </select>
        </p>
  
        {% if not run_as_a_service %}
        <p>Operation will be performed using the selected queue:
          {% if queues %}
          <select name="queue">
            {% for queue in queues %}
              <option value="{{ queue }}">{{ queue }}</option>
            {% endfor %}
          </select>
          {% else %}
          <input type="text" name="queue" value="default"/>
          {% endif %}
        </p>
        {% endif %}
  
        <table>
          <tr>
            <td>
              Backup storage destination:
              <select name="filesystem" id="ae-datastore-admin-filesystem">
                {% if not run_as_a_service and not hide_blobstore %}
                <option value="blobstore">Blobstore</option>
                {% endif %}
                <option value="gs" selected="selected">Google Cloud Storage</option>
              </select>
            </td>
          </tr>
          <tr id="gs_bucket_tr">
            <td>
              Google Cloud Storage bucket name
              (<a target="_blank"
                href="https://cloud.google.com/datastore/docs/console/datastore-backing-up-restoring">More info?</a>):
              <input type="text" id="gs_bucket_name" name="gs_bucket_name" value="" />
            </td>
          </tr>
          <tr id="gs_service_account_tr">
            <td>
              Service account name
              <img class="ae-help-icon" src="{{ base_path }}/static/img/help.gif" height="14" width="14" alt="help"
                   title="Use the client_email field from any custom service accounts you've defined.">
              <input name="service_account_name">
            </td>
          </tr>
        </table>
  
        <table style="margin-top: 1em;"><tr>
          <td style="padding-right: 0.5em;">
            <input class="go
  
  [... the rest is omitted ...]
files.new_path: AppServer/google/appengine/ext/datastore_admin/templates/confirm_backup.html
files.old_contents: {% extends "base.html" %}
  {% block title %}Confirm Backup of {{kind_str|escape}}{% endblock %}
  {% block body %}
    <h2>Datastore Admin: Backup to Google Cloud Storage</h2>
    {% if kind_list %}
      {% comment %}
      size_total represents the total size of the figures for which we have
      datastore statistics available.
      sizes_known indicates whether or not we have datastore statistics for all
      kinds.
      These two variables are used to render different messages for the three
      possible scenarios: all sizes known, some sizes known, no sizes known.
      {% endcomment %}
  
      <form class="ae-form" action="{{base_path}}/{{form_target}}" method="post"
          style="width:39.39em;">
        {% if run_as_a_service %}
          <input type="hidden" name="run_as_a_service" value="{{ run_as_a_service|escape }}">
        {% endif %}
        {% for kind in kind_list %}
        <input type="hidden" name="kind" value="{{kind|escape}}">
        {% endfor %}
  
        <div class="ae-input-row">
          <div class="ae-label-row">
            <label for="backup_name">Backup name: </label>
          </div>
          <div class="ae-input-row">
            <input type="text" id="backup_name" name="backup_name"
                style="width:100%"
                value="{{backup_name}}">
          </div>
        </div>
        <input type="hidden" name="xsrf_token" value="{{xsrf_token|escape}}">
  
        {% if notreadonly_warning %}
        <div>
          <h4 style="padding:0;margin-top:1em;margin-bottom:0.46em;">WARNING</h4>
          <p>The application is not in read-only mode and updates
          during the backup process may not be captured.
          To place the application in read-only mode,
          please follow the instructions at
          <a href="https://cloud.google.com/datastore/docs/console/managing-datastore#enabling_datastore_writes" target="_blank">
          Enabling or Disabling Datastore Writes</a>
          </p>
        </div>
        {% endif %}
  
        {% if blob_warning %}
        <p>The application has blob data.  BlobInfo records and data will NOT be
        backed up.</p>
        {% endif %}
  
        <p>Selected namespace:
          <select name="namespace">
            {% for namespace in namespaces %}
              <option value="{{ namespace.1 }}" {% if namespace.2 %}selected="selected"{% endif %}>{{ namespace.0 }}</option>
            {% endfor %}
          </select>
        </p>
  
        {% if not run_as_a_service %}
        <p>Operation will be performed using the selected queue:
          {% if queues %}
          <select name="queue">
            {% for queue in queues %}
              <option value="{{ queue }}">{{ queue }}</option>
            {% endfor %}
          </select>
          {% else %}
          <input type="text" name="queue" value="default"/>
          {% endif %}
        </p>
        {% endif %}
  
        <table>
          <tr>
            <td>
              Backup storage destination:
              <select name="filesystem" id="ae-datastore-admin-filesystem">
                {% if not run_as_a_service and not hide_blobstore %}
                <option value="blobstore">Blobstore</option>
                {% endif %}
                <option value="gs" selected="selected">Google Cloud Storage</option>
              </select>
            </td>
          </tr>
          <tr id="gs_bucket_tr">
            <td>
              Google Cloud Storage bucket name
              (<a target="_blank"
                href="https://cloud.google.com/datastore/docs/console/datastore-backing-up-restoring">More info?</a>):
              <input type="text" id="gs_bucket_name" name="gs_bucket_name" value="" />
            </td>
          </tr>
        </table>
  
        <table style="margin-top: 1em;"><tr>
          <td style="padding-right: 0.5em;">
            <input class="goog-button" type="submit" name="backup"
                value="Backup Entities"></td>
          <td style="color: #666667;">
            or <a href="{{ datastore_admin_home }}">Cancel</a></td>
        </tr></table>
      </form>
    {% else %}
      <div class="ae-errorbox">No entity kinds to backup.</div>
    {% endif %}
  
  
  [... the rest is omitted ...]
files.old_path: AppServer/google/appengine/ext/datastore_admin/templates/confirm_backup.html