python/google/appengine/ext/remote_api/handler.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2007 Google Inc.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #
  17
  18
  19
  20
  21 """A handler that exports various App Engine services over HTTP.
  22
  23 You can export this handler in your app by adding it to the builtins section:
  24
  25 builtins:
  26 - remote_api: on
  27
  28 This will add remote_api serving to the path /_ah/remote_api.
  29
  30 You can also add it to your handlers section, e.g.:
  31
  32   handlers:
  33   - url: /remote_api(/.*)?
  34     script: $PYTHON_LIB/google/appengine/ext/remote_api/handler.py
  35
  36 You can use remote_api_stub to remotely access services exported by this
  37 handler. See the documentation in remote_api_stub.py for details on how to do
  38 this.
  39
  40 The handler supports several forms of authentication. By default, it
  41 checks that the user is an admin using the Users API, similar to specifying
  42 "login: admin" in the app.yaml file. It also supports a 'custom header' mode
  43 which can be used in certain scenarios.
  44
  45 To configure the custom header mode, edit an appengine_config file (the same
  46 one you may use to configure appstats) to include a line like this:
  47
  48   remoteapi_CUSTOM_ENVIRONMENT_AUTHENTICATION = (
  49       'HTTP_X_APPENGINE_INBOUND_APPID', ['otherappid'] )
  50
  51 See the ConfigDefaults class below for the full set of options available.
  52 """
  53
  54
  55
  56
  57
  58
  59
  60
  61
  62 import google
  63 import hashlib
  64 import logging
  65 import os
  66 import pickle
  67 import wsgiref.handlers
  68 import yaml
  69
  70 from google.appengine.api import api_base_pb
  71 from google.appengine.api import apiproxy_stub
  72 from google.appengine.api import apiproxy_stub_map
  73 from google.appengine.api import datastore_types
  74 from google.appengine.api import lib_config
  75 from google.appengine.api import oauth
  76 from google.appengine.api import users
  77 from google.appengine.datastore import datastore_pb
  78 from google.appengine.datastore import datastore_rpc
  79 from google.appengine.ext import webapp
  80 from google.appengine.ext.db import metadata
  81 from google.appengine.ext.remote_api import remote_api_pb
  82 from google.appengine.ext.remote_api import remote_api_services
  83 from google.appengine.runtime import apiproxy_errors
  84 from google.appengine.datastore import entity_pb
  85
  86
  87 class ConfigDefaults(object):
  88   """Configurable constants.
  89
  90   To override remote_api configuration values, define values like this
  91   in your appengine_config.py file (in the root of your app):
  92
  93     remoteapi_CUSTOM_ENVIRONMENT_AUTHENTICATION = (
  94         'HTTP_X_APPENGINE_INBOUND_APPID', ['otherappid'] )
  95
  96   You may wish to base this file on sample_appengine_config.py.
  97   """
  98
  99   # Allow other App Engine applications to use remote_api with special forms
 100   # of authentication which appear in the environment. This is a pair,
 101   # ( environment variable name, [ list of valid values ] ). Some examples:
 102   # * Allow other applications to use remote_api:
 103   #   remoteapi_CUSTOM_ENVIRONMENT_AUTHENTICATION = (
 104   #       'HTTP_X_APPENGINE_INBOUND_APPID', ['otherappid'] )
 105   # * Allow two specific users (who need not be admins):
 106   #   remoteapi_CUSTOM_ENVIRONMENT_AUTHENTICATION = ('USER_ID',
 107   #                                                  [ '1234', '1111' ] )
 108
 109
 110
 111
 112
 113   # Note that this an alternate to the normal users.is_current_user_admin
 114   # check--either one may pass.
 115   CUSTOM_ENVIRONMENT_AUTHENTICATION = ()
 116   _ALLOW_OAUTH = False
 117
 118
 119 config = lib_config.register('remoteapi', ConfigDefaults.__dict__)
 120
 121
 122 class RemoteDatastoreStub(apiproxy_stub.APIProxyStub):
 123   """Provides a stub that permits execution of stateful datastore queries.
 124
 125   Some operations aren't possible using the standard interface. Notably,
 126   datastore RunQuery operations internally store a cursor that is referenced in
 127   later Next calls, and cleaned up at the end of each request. Because every
 128   call to ApiCallHandler takes place in its own request, this isn't possible.
 129
 130   To work around this, RemoteDatastoreStub provides its own implementation of
 131   RunQuery that immediately returns the query results.
 132   """
 133
 134   def __init__(self, service='datastore_v3', _test_stub_map=None):
 135     """Constructor.
 136
 137     Args:
 138       service: The name of the service
 139       _test_stub_map: An APIProxyStubMap to use for testing purposes.
 140     """
 141     super(RemoteDatastoreStub, self).__init__(service)
 142     if _test_stub_map:
 143       self.__call = _test_stub_map.MakeSyncCall
 144     else:
 145       self.__call = apiproxy_stub_map.MakeSyncCall
 146
 147   def _Dynamic_RunQuery(self, request, response):
 148     """Handle a RunQuery request.
 149
 150     We handle RunQuery by executing a Query and a Next and returning the result
 151     of the Next request.
 152
 153     This method is DEPRECATED, but left in place for older clients.
 154     """
 155     runquery_response = datastore_pb.QueryResult()
 156     self.__call('datastore_v3', 'RunQuery', request, runquery_response)
 157     if runquery_response.result_size() > 0:
 158
 159       response.CopyFrom(runquery_response)
 160       return
 161
 162
 163     next_request = datastore_pb.NextRequest()
 164     next_request.mutable_cursor().CopyFrom(runquery_response.cursor())
 165     next_request.set_count(request.limit())
 166     self.__call('datastore_v3', 'Next', next_request, response)
 167
 168   def _Dynamic_TransactionQuery(self, request, response):
 169     if not request.has_ancestor():
 170       raise apiproxy_errors.ApplicationError(
 171           datastore_pb.Error.BAD_REQUEST,
 172           'No ancestor in transactional query.')
 173
 174     app_id = datastore_types.ResolveAppId(None)
 175     if (datastore_rpc._GetDatastoreType(app_id) !=
 176         datastore_rpc.BaseConnection.HIGH_REPLICATION_DATASTORE):
 177       raise apiproxy_errors.ApplicationError(
 178           datastore_pb.Error.BAD_REQUEST,
 179           'remote_api supports transactional queries only in the '
 180           'high-replication datastore.')
 181
 182
 183     entity_group_key = entity_pb.Reference()
 184     entity_group_key.CopyFrom(request.ancestor())
 185     group_path = entity_group_key.mutable_path()
 186     root = entity_pb.Path_Element()
 187     root.MergeFrom(group_path.element(0))
 188     group_path.clear_element()
 189     group_path.add_element().CopyFrom(root)
 190     eg_element = group_path.add_element()
 191     eg_element.set_type(metadata.EntityGroup.KIND_NAME)
 192     eg_element.set_id(metadata.EntityGroup.ID)
 193
 194
 195     begin_request = datastore_pb.BeginTransactionRequest()
 196     begin_request.set_app(app_id)
 197     tx = datastore_pb.Transaction()
 198     self.__call('datastore_v3', 'BeginTransaction', begin_request, tx)
 199
 200
 201     request.mutable_transaction().CopyFrom(tx)
 202     self.__call('datastore_v3', 'RunQuery', request, response.mutable_result())
 203
 204
 205     get_request = datastore_pb.GetRequest()
 206     get_request.mutable_transaction().CopyFrom(tx)
 207     get_request.add_key().CopyFrom(entity_group_key)
 208     get_response = datastore_pb.GetResponse()
 209     self.__call('datastore_v3', 'Get', get_request, get_response)
 210     entity_group = get_response.entity(0)
 211
 212
 213     response.mutable_entity_group_key().CopyFrom(entity_group_key)
 214     if entity_group.has_entity():
 215       response.mutable_entity_group().CopyFrom(entity_group.entity())
 216
 217
 218     self.__call('datastore_v3', 'Commit', tx, datastore_pb.CommitResponse())
 219
 220   def _Dynamic_Transaction(self, request, response):
 221     """Handle a Transaction request.
 222
 223     We handle transactions by accumulating Put and Delete requests on the client
 224     end, as well as recording the key and hash of Get requests. When Commit is
 225     called, Transaction is invoked, which verifies that all the entities in the
 226     precondition list still exist and their hashes match, then performs a
 227     transaction of its own to make the updates.
 228     """
 229
 230     begin_request = datastore_pb.BeginTransactionRequest()
 231     begin_request.set_app(os.environ['APPLICATION_ID'])
 232     begin_request.set_allow_multiple_eg(request.allow_multiple_eg())
 233     tx = datastore_pb.Transaction()
 234     self.__call('datastore_v3', 'BeginTransaction', begin_request, tx)
 235
 236
 237     preconditions = request.precondition_list()
 238     if preconditions:
 239       get_request = datastore_pb.GetRequest()
 240       get_request.mutable_transaction().CopyFrom(tx)
 241       for precondition in preconditions:
 242         key = get_request.add_key()
 243         key.CopyFrom(precondition.key())
 244       get_response = datastore_pb.GetResponse()
 245       self.__call('datastore_v3', 'Get', get_request, get_response)
 246       entities = get_response.entity_list()
 247       assert len(entities) == request.precondition_size()
 248       for precondition, entity in zip(preconditions, entities):
 249         if precondition.has_hash() != entity.has_entity():
 250           raise apiproxy_errors.ApplicationError(
 251               datastore_pb.Error.CONCURRENT_TRANSACTION,
 252               "Transaction precondition failed.")
 253         elif entity.has_entity():
 254           entity_hash = hashlib.sha1(entity.entity().Encode()).digest()
 255           if precondition.hash() != entity_hash:
 256             raise apiproxy_errors.ApplicationError(
 257                 datastore_pb.Error.CONCURRENT_TRANSACTION,
 258                 "Transaction precondition failed.")
 259
 260
 261     if request.has_puts():
 262       put_request = request.puts()
 263       put_request.mutable_transaction().CopyFrom(tx)
 264       self.__call('datastore_v3', 'Put', put_request, response)
 265
 266
 267     if request.has_deletes():
 268       delete_request = request.deletes()
 269       delete_request.mutable_transaction().CopyFrom(tx)
 270       self.__call('datastore_v3', 'Delete', delete_request,
 271                   datastore_pb.DeleteResponse())
 272
 273
 274     self.__call('datastore_v3', 'Commit', tx, datastore_pb.CommitResponse())
 275
 276   def _Dynamic_GetIDsXG(self, request, response):
 277     self._Dynamic_GetIDs(request, response, is_xg=True)
 278
 279   def _Dynamic_GetIDs(self, request, response, is_xg=False):
 280     """Fetch unique IDs for a set of paths."""
 281
 282     for entity in request.entity_list():
 283       assert entity.property_size() == 0
 284       assert entity.raw_property_size() == 0
 285       assert entity.entity_group().element_size() == 0
 286       lastpart = entity.key().path().element_list()[-1]
 287       assert lastpart.id() == 0 and not lastpart.has_name()
 288
 289
 290     begin_request = datastore_pb.BeginTransactionRequest()
 291     begin_request.set_app(os.environ['APPLICATION_ID'])
 292     begin_request.set_allow_multiple_eg(is_xg)
 293     tx = datastore_pb.Transaction()
 294     self.__call('datastore_v3', 'BeginTransaction', begin_request, tx)
 295
 296
 297     self.__call('datastore_v3', 'Put', request, response)
 298
 299
 300     self.__call('datastore_v3', 'Rollback', tx, api_base_pb.VoidProto())
 301
 302
 303
 304 SERVICE_PB_MAP = remote_api_services.SERVICE_PB_MAP
 305
 306 class ApiCallHandler(webapp.RequestHandler):
 307   """A webapp handler that accepts API calls over HTTP and executes them."""
 308
 309   LOCAL_STUBS = {
 310       'remote_datastore': RemoteDatastoreStub('remote_datastore'),
 311   }
 312
 313   OAUTH_SCOPE = 'https://www.googleapis.com/auth/appengine.apis'
 314
 315   def CheckIsAdmin(self):
 316     user_is_authorized = False
 317     if users.is_current_user_admin():
 318       user_is_authorized = True
 319     if not user_is_authorized and config.CUSTOM_ENVIRONMENT_AUTHENTICATION:
 320       if len(config.CUSTOM_ENVIRONMENT_AUTHENTICATION) == 2:
 321         var, values = config.CUSTOM_ENVIRONMENT_AUTHENTICATION
 322         if os.getenv(var) in values:
 323           user_is_authorized = True
 324       else:
 325         logging.warning('remoteapi_CUSTOM_ENVIRONMENT_AUTHENTICATION is '
 326                         'configured incorrectly.')
 327
 328     if not user_is_authorized and config._ALLOW_OAUTH:
 329       try:
 330         user_is_authorized = (
 331             oauth.is_current_user_admin(_scope=self.OAUTH_SCOPE))
 332       except oauth.OAuthRequestError:
 333
 334         pass
 335     if not user_is_authorized:
 336       self.response.set_status(401)
 337       self.response.out.write(
 338           'You must be logged in as an administrator to access this.')
 339       self.response.headers['Content-Type'] = 'text/plain'
 340       return False
 341     if 'X-appcfg-api-version' not in self.request.headers:
 342       self.response.set_status(403)
 343       self.response.out.write('This request did not contain a necessary header')
 344       self.response.headers['Content-Type'] = 'text/plain'
 345       return False
 346     return True
 347
 348   def CheckConfigIsValid(self):
 349
 350     if config.CUSTOM_ENVIRONMENT_AUTHENTICATION and config._ALLOW_OAUTH:
 351       self.response.set_status(400)
 352       self.response.out.write('You cannot enable both OAuth authentication '
 353                               '(remoteapi__ALLOW_OAUTH) and '
 354                               'custom authentication '
 355                               '(remoteapi_CUSTOM_ENVIRONMENT_AUTHENTICATION).')
 356       return False
 357     return True
 358
 359
 360   def get(self):
 361     """Handle a GET. Just show an info page."""
 362     if not self.CheckConfigIsValid() or not self.CheckIsAdmin():
 363       return
 364
 365     rtok = self.request.get('rtok', '0')
 366     app_info = {
 367         'app_id': os.environ['APPLICATION_ID'],
 368         'rtok': rtok
 369         }
 370
 371     self.response.headers['Content-Type'] = 'text/plain'
 372     self.response.out.write(yaml.dump(app_info))
 373
 374   def post(self):
 375     """Handle POST requests by executing the API call."""
 376     if not self.CheckConfigIsValid() or not self.CheckIsAdmin():
 377       return
 378
 379
 380
 381
 382
 383
 384
 385
 386
 387
 388
 389
 390
 391
 392
 393
 394     self.response.headers['Content-Type'] = 'text/plain'
 395
 396     response = remote_api_pb.Response()
 397     try:
 398       request = remote_api_pb.Request()
 399
 400
 401
 402       request.ParseFromString(self.request.body)
 403       response_data = self.ExecuteRequest(request)
 404       response.set_response(response_data.Encode())
 405       self.response.set_status(200)
 406     except Exception, e:
 407       logging.exception('Exception while handling %s', request)
 408       self.response.set_status(200)
 409
 410
 411
 412       response.set_exception(pickle.dumps(e))
 413       if isinstance(e, apiproxy_errors.ApplicationError):
 414         application_error = response.mutable_application_error()
 415         application_error.set_code(e.application_error)
 416         application_error.set_detail(e.error_detail)
 417     self.response.out.write(response.Encode())
 418
 419   def ExecuteRequest(self, request):
 420     """Executes an API invocation and returns the response object."""
 421     service = request.service_name()
 422     method = request.method()
 423     service_methods = SERVICE_PB_MAP.get(service, {})
 424     request_class, response_class = service_methods.get(method, (None, None))
 425     if not request_class:
 426       raise apiproxy_errors.CallNotFoundError()
 427
 428     request_data = request_class()
 429     request_data.ParseFromString(request.request())
 430     response_data = response_class()
 431
 432     if service in self.LOCAL_STUBS:
 433       self.LOCAL_STUBS[service].MakeSyncCall(service, method, request_data,
 434                                              response_data)
 435     else:
 436       apiproxy_stub_map.MakeSyncCall(service, method, request_data,
 437                                      response_data)
 438
 439     return response_data
 440
 441   def InfoPage(self):
 442     """Renders an information page."""
 443     return """
 444 <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN"
 445  "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
 446 <html><head>
 447 <title>App Engine API endpoint.</title>
 448 </head><body>
 449 <h1>App Engine API endpoint.</h1>
 450 <p>This is an endpoint for the App Engine remote API interface.
 451 Point your stubs (google.appengine.ext.remote_api.remote_api_stub) here.</p>
 452 </body>
 453 </html>"""
 454
 455 application = webapp.WSGIApplication([('.*', ApiCallHandler)])
 456
 457
 458 def main():
 459   wsgiref.handlers.CGIHandler().run(application)
 460
 461
 462 if __name__ == '__main__':
 463   main()