python/google/appengine/ext/datastore_admin/copy_handler.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2007 Google Inc.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #
  17
  18
  19
  20
  21
  22 """Handler for data copy operation.
  23
  24 Generic datastore admin console transfers control to ConfirmCopyHandler
  25 after selection of entities. The ConfirmCopyHandler confirms with user
  26 his choice, enters target application id and transfers control to
  27 DoCopyHandler. DoCopyHandler starts copying mappers and displays confirmation
  28 page.
  29
  30 This module also contains actual mapper code for copying data over.
  31 """
  32
  33
  34 import logging
  35 import urllib
  36
  37 from google.appengine.api import capabilities
  38 from google.appengine.api import datastore
  39 from google.appengine.datastore import datastore_rpc
  40 from google.appengine.ext import blobstore
  41 from google.appengine.ext import webapp
  42 from google.appengine.ext.datastore_admin import remote_api_put_stub
  43 from google.appengine.ext.datastore_admin import utils
  44 from google.appengine.ext.mapreduce import context
  45 from google.appengine.ext.mapreduce import operation
  46
  47
  48 XSRF_ACTION = 'copy'
  49
  50
  51 class ConfirmCopyHandler(webapp.RequestHandler):
  52   """Handler to deal with requests from the admin console to copy data."""
  53
  54   SUFFIX = 'confirm_copy'
  55
  56   @classmethod
  57   def Render(cls, handler):
  58     """Rendering method that can be called by main.py.
  59
  60     Args:
  61       handler: the webapp.RequestHandler invoking the method
  62     """
  63     namespace = handler.request.get('namespace')
  64     kinds = handler.request.get_all('kind')
  65     sizes_known, size_total, remainder = utils.ParseKindsAndSizes(kinds)
  66
  67     (namespace_str, kind_str) = utils.GetPrintableStrs(namespace, kinds)
  68     notreadonly_warning = capabilities.CapabilitySet(
  69         'datastore_v3', capabilities=['write']).is_enabled()
  70     blob_warning = bool(blobstore.BlobInfo.all().fetch(1))
  71     datastore_type = datastore._GetConnection().get_datastore_type()
  72     high_replication_warning = (
  73         datastore_type == datastore_rpc.Connection.HIGH_REPLICATION_DATASTORE)
  74
  75     template_params = {
  76         'form_target': DoCopyHandler.SUFFIX,
  77         'kind_list': kinds,
  78         'remainder': remainder,
  79         'sizes_known': sizes_known,
  80         'size_total': size_total,
  81         'app_id': handler.request.get('app_id'),
  82         'cancel_url': handler.request.get('cancel_url'),
  83         'kind_str': kind_str,
  84         'namespace_str': namespace_str,
  85         'xsrf_token': utils.CreateXsrfToken(XSRF_ACTION),
  86         'notreadonly_warning': notreadonly_warning,
  87         'blob_warning': blob_warning,
  88         'high_replication_warning': high_replication_warning,
  89     }
  90     utils.RenderToResponse(handler, 'confirm_copy.html', template_params)
  91
  92
  93
  94
  95 class DoCopyHandler(webapp.RequestHandler):
  96   """Handler to deal with requests from the admin console to copy data."""
  97
  98   SUFFIX = 'copy.do'
  99
 100   COPY_HANDLER = ('google.appengine.ext.datastore_admin.copy_handler.'
 101                   'RemoteCopyEntity.map')
 102   INPUT_READER = ('google.appengine.ext.mapreduce.input_readers.'
 103                   'ConsistentKeyReader')
 104   MAPREDUCE_DETAIL = utils.config.MAPREDUCE_PATH + '/detail?mapreduce_id='
 105
 106   def get(self):
 107     """Handler for get requests to datastore_admin/copy.do.
 108
 109     Status of executed jobs is displayed.
 110     """
 111     jobs = self.request.get_all('job')
 112     error = self.request.get('error', '')
 113     xsrf_error = self.request.get('xsrf_error', '')
 114
 115     template_params = {
 116         'job_list': jobs,
 117         'mapreduce_detail': self.MAPREDUCE_DETAIL,
 118         'error': error,
 119         'xsrf_error': xsrf_error,
 120         'datastore_admin_home': utils.config.BASE_PATH,
 121     }
 122     utils.RenderToResponse(self, 'do_copy.html', template_params)
 123
 124   def post(self):
 125     """Handler for post requests to datastore_admin/copy.do.
 126
 127     Jobs are executed and user is redirected to the get handler.
 128     """
 129     namespace = self.request.get('namespace')
 130     kinds = self.request.get_all('kind')
 131     (namespace_str, kinds_str) = utils.GetPrintableStrs(namespace, kinds)
 132     token = self.request.get('xsrf_token')
 133     remote_url = self.request.get('remote_url')
 134     extra_header = self.request.get('extra_header')
 135
 136     jobs = []
 137     if not remote_url:
 138       parameters = [('error', 'Unspecified remote URL.')]
 139     elif not utils.ValidateXsrfToken(token, XSRF_ACTION):
 140       parameters = [('xsrf_error', '1')]
 141     else:
 142       try:
 143
 144
 145         if extra_header:
 146           extra_headers = dict([extra_header.split(':', 1)])
 147         else:
 148           extra_headers = None
 149         target_app = remote_api_put_stub.get_remote_appid(remote_url,
 150                                                           extra_headers)
 151         op = utils.StartOperation(
 152             'Copying %s%s to %s' % (kinds_str, namespace_str, target_app))
 153         name_template = 'Copy all %(kind)s objects%(namespace)s'
 154         mapper_params = {
 155             'target_app': target_app,
 156             'remote_url': remote_url,
 157             'extra_header': extra_header,
 158         }
 159         jobs = utils.RunMapForKinds(
 160             op.key(),
 161             kinds,
 162             name_template,
 163             self.COPY_HANDLER,
 164             self.INPUT_READER,
 165             None,
 166             mapper_params)
 167
 168         error = ''
 169
 170
 171       except Exception, e:
 172         logging.exception('Handling exception.')
 173         error = self._HandleException(e)
 174
 175       parameters = [('job', job) for job in jobs]
 176       if error:
 177         parameters.append(('error', error))
 178
 179     query = urllib.urlencode(parameters)
 180     self.redirect('%s/%s?%s' % (utils.config.BASE_PATH, self.SUFFIX, query))
 181
 182   def _HandleException(self, e):
 183     """Make exception handling overrideable by tests.
 184
 185     In normal cases, return only the error string; do not fail to render the
 186     page for user.
 187     """
 188     return str(e)
 189
 190
 191
 192 def KindPathFromKey(key):
 193   """Return kinds path as '/'-delimited string for a particular key."""
 194   path = key.to_path()
 195   kinds = []
 196   is_kind = True
 197   for item in path:
 198     if is_kind:
 199       kinds.append(item)
 200     is_kind = not is_kind
 201   kind_path = '/'.join(kinds)
 202   return kind_path
 203
 204
 205 def get_mapper_params():
 206   """Return current mapreduce mapper params. Easily stubbed out for testing."""
 207   return context.get().mapreduce_spec.mapper.params
 208
 209
 210 class CopyEntity(object):
 211   """A class which contains a map handler to copy entities."""
 212
 213   def map(self, key):
 214     """Copy data map handler.
 215
 216     Args:
 217       key: Datastore entity key or entity itself to copy.
 218
 219     Yields:
 220       A db operation to store the entity in the target app.
 221       An operation which updates max used ID if necessary.
 222       A counter operation incrementing the count for the entity kind.
 223     """
 224
 225     mapper_params = get_mapper_params()
 226     target_app = mapper_params['target_app']
 227
 228     if isinstance(key, datastore.Entity):
 229
 230       entity = key
 231       key = entity.key()
 232     else:
 233       entity = datastore.Get(key)
 234     entity_proto = entity._ToPb()
 235     utils.FixKeys(entity_proto, target_app)
 236     target_entity = datastore.Entity._FromPb(entity_proto)
 237
 238     yield operation.db.Put(target_entity)
 239     yield utils.ReserveKey(key, target_app)
 240     yield operation.counters.Increment(KindPathFromKey(key))
 241
 242
 243 class RemoteCopyEntity(CopyEntity):
 244   """A class which contains a map handler to copy entities remotely.
 245
 246   The class manages the connection.
 247   """
 248
 249   def __init__(self):
 250     super(RemoteCopyEntity, self).__init__()
 251     self.remote_api_stub_initialized = False
 252
 253   def setup_stub(self):
 254     """Set up the remote API stub."""
 255     if self.remote_api_stub_initialized:
 256       return
 257     params = get_mapper_params()
 258     if 'extra_header' in params and params['extra_header']:
 259
 260       extra_headers = dict([params['extra_header'].split(':', 1)])
 261     else:
 262       extra_headers = {}
 263
 264     remote_api_put_stub.configure_remote_put(params['remote_url'],
 265                                              params['target_app'],
 266                                              extra_headers)
 267
 268     self.remote_api_stub_initialized = True
 269
 270   def map(self, key):
 271     """Copy data map handler.
 272
 273     Args:
 274       key: Datastore entity key to copy.
 275
 276     Yields:
 277       A db operation to store the entity in the target app.
 278       An operation which updates max used ID if necessary.
 279       A counter operation incrementing the count for the entity kind.
 280     """
 281     if not self.remote_api_stub_initialized:
 282       self.setup_stub()
 283
 284     for op in CopyEntity.map(self, key):
 285       yield op
 286
 287
 288 def handlers_list(base_path):
 289   return [
 290       (r'%s/%s' % (base_path, ConfirmCopyHandler.SUFFIX), ConfirmCopyHandler),
 291       (r'%s/%s' % (base_path, DoCopyHandler.SUFFIX), DoCopyHandler),
 292       ]