python/google/appengine/tools/devappserver2/api_server.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2007 Google Inc.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #
  17 """Serves the stub App Engine APIs (e.g. memcache, datastore) over HTTP.
  18
  19 The Remote API protocol is used for communication.
  20 """
  21
  22
  23 import logging
  24 import os
  25 import pickle
  26 import shutil
  27 import socket
  28 import sys
  29 import tempfile
  30 import threading
  31 import time
  32 import traceback
  33 import urllib2
  34 import urlparse
  35
  36 import google
  37 import yaml
  38
  39 # Stubs
  40 from google.appengine.api import datastore_file_stub
  41 from google.appengine.api import mail_stub
  42 from google.appengine.api import urlfetch_stub
  43 from google.appengine.api import user_service_stub
  44 from google.appengine.api.app_identity import app_identity_stub
  45 from google.appengine.api.blobstore import blobstore_stub
  46 from google.appengine.api.blobstore import file_blob_storage
  47 from google.appengine.api.capabilities import capability_stub
  48 from google.appengine.api.channel import channel_service_stub
  49 from google.appengine.api.files import file_service_stub
  50 from google.appengine.api.logservice import logservice_stub
  51 from google.appengine.api.search import simple_search_stub
  52 from google.appengine.api.taskqueue import taskqueue_stub
  53 from google.appengine.api.prospective_search import prospective_search_stub
  54 from google.appengine.api.memcache import memcache_stub
  55 from google.appengine.api.modules import modules_stub
  56 from google.appengine.api.remote_socket import _remote_socket_stub
  57 from google.appengine.api.system import system_stub
  58 from google.appengine.api.xmpp import xmpp_service_stub
  59 from google.appengine.datastore import datastore_sqlite_stub
  60 from google.appengine.datastore import datastore_stub_util
  61 from google.appengine.datastore import datastore_v4_pb
  62 from google.appengine.datastore import datastore_v4_stub
  63
  64 from google.appengine.api import apiproxy_stub_map
  65 from google.appengine.api import datastore
  66 from google.appengine.ext.remote_api import remote_api_pb
  67 from google.appengine.ext.remote_api import remote_api_services
  68 from google.appengine.runtime import apiproxy_errors
  69 from google.appengine.tools.devappserver2 import wsgi_server
  70
  71
  72 # TODO: Remove this lock when stubs have been audited for thread
  73 # safety.
  74 GLOBAL_API_LOCK = threading.RLock()
  75
  76
  77 # We don't want to support datastore_v4 everywhere, because users are supposed
  78 # to use the Cloud Datastore API going forward, so we don't want to put these
  79 # entries in remote_api_servers.SERVICE_PB_MAP. But for our own implementation
  80 # of the Cloud Datastore API we need those methods to work when an instance
  81 # issues them, specifically the DatstoreApiServlet running as a module inside
  82 # the app we are running. The consequence is that other app code can also
  83 # issue datastore_v4 API requests, but since we don't document these requests
  84 # or export them through any language bindings this is unlikely in practice.
  85 _DATASTORE_V4_METHODS = {
  86     'AllocateIds': (datastore_v4_pb.AllocateIdsRequest,
  87                     datastore_v4_pb.AllocateIdsResponse),
  88     'BeginTransaction': (datastore_v4_pb.BeginTransactionRequest,
  89                          datastore_v4_pb.BeginTransactionResponse),
  90     'Commit': (datastore_v4_pb.CommitRequest,
  91                datastore_v4_pb.CommitResponse),
  92     'ContinueQuery': (datastore_v4_pb.ContinueQueryRequest,
  93                       datastore_v4_pb.ContinueQueryResponse),
  94     'Lookup': (datastore_v4_pb.LookupRequest,
  95                datastore_v4_pb.LookupResponse),
  96     'Rollback': (datastore_v4_pb.RollbackRequest,
  97                  datastore_v4_pb.RollbackResponse),
  98     'RunQuery': (datastore_v4_pb.RunQueryRequest,
  99                  datastore_v4_pb.RunQueryResponse),
 100 }
 101
 102
 103 def _execute_request(request):
 104   """Executes an API method call and returns the response object.
 105
 106   Args:
 107     request: A remote_api_pb.Request object representing the API call e.g. a
 108         call to memcache.Get.
 109
 110   Returns:
 111     A ProtocolBuffer.ProtocolMessage representing the API response e.g. a
 112     memcache_service_pb.MemcacheGetResponse.
 113
 114   Raises:
 115     apiproxy_errors.CallNotFoundError: if the requested method doesn't exist.
 116     apiproxy_errors.ApplicationError: if the API method calls fails.
 117   """
 118   service = request.service_name()
 119   method = request.method()
 120   if request.has_request_id():
 121     request_id = request.request_id()
 122   else:
 123     logging.error('Received a request without request_id: %s', request)
 124     request_id = None
 125
 126   service_methods = (_DATASTORE_V4_METHODS if service == 'datastore_v4'
 127                      else remote_api_services.SERVICE_PB_MAP.get(service, {}))
 128   # We do this rather than making a new map that is a superset of
 129   # remote_api_services.SERVICE_PB_MAP because that map is not initialized
 130   # all in one place, so we would have to be careful about where we made
 131   # our new map.
 132
 133   request_class, response_class = service_methods.get(method, (None, None))
 134   if not request_class:
 135     raise apiproxy_errors.CallNotFoundError('%s.%s does not exist' % (service,
 136                                                                       method))
 137
 138   request_data = request_class()
 139   request_data.ParseFromString(request.request())
 140   response_data = response_class()
 141   service_stub = apiproxy_stub_map.apiproxy.GetStub(service)
 142
 143   def make_request():
 144     service_stub.MakeSyncCall(service,
 145                               method,
 146                               request_data,
 147                               response_data,
 148                               request_id)
 149
 150   # If the service has not declared itself as threadsafe acquire
 151   # GLOBAL_API_LOCK.
 152   if service_stub.THREADSAFE:
 153     make_request()
 154   else:
 155     with GLOBAL_API_LOCK:
 156       make_request()
 157   return response_data
 158
 159
 160 class APIServer(wsgi_server.WsgiServer):
 161   """Serves API calls over HTTP."""
 162
 163   def __init__(self, host, port, app_id):
 164     self._app_id = app_id
 165     self._host = host
 166     super(APIServer, self).__init__((host, port), self)
 167
 168   def start(self):
 169     """Start the API Server."""
 170     super(APIServer, self).start()
 171     logging.info('Starting API server at: http://%s:%d', self._host, self.port)
 172
 173   def quit(self):
 174     cleanup_stubs()
 175     super(APIServer, self).quit()
 176
 177   def _handle_POST(self, environ, start_response):
 178     start_response('200 OK', [('Content-Type', 'application/octet-stream')])
 179
 180     start_time = time.time()
 181     response = remote_api_pb.Response()
 182     try:
 183       request = remote_api_pb.Request()
 184       # NOTE: Exceptions encountered when parsing the PB or handling the request
 185       # will be propagated back to the caller the same way as exceptions raised
 186       # by the actual API call.
 187       if environ.get('HTTP_TRANSFER_ENCODING') == 'chunked':
 188         # CherryPy concatenates all chunks  when 'wsgi.input' is read but v3.2.2
 189         # will not return even when all of the data in all chunks has been
 190         # read. See: https://bitbucket.org/cherrypy/cherrypy/issue/1131.
 191         wsgi_input = environ['wsgi.input'].read(2**32)
 192       else:
 193         wsgi_input = environ['wsgi.input'].read(int(environ['CONTENT_LENGTH']))
 194       request.ParseFromString(wsgi_input)
 195       api_response = _execute_request(request).Encode()
 196       response.set_response(api_response)
 197     except Exception, e:
 198       if isinstance(e, apiproxy_errors.ApplicationError):
 199         level = logging.DEBUG
 200         application_error = response.mutable_application_error()
 201         application_error.set_code(e.application_error)
 202         application_error.set_detail(e.error_detail)
 203       else:
 204         # If the runtime instance is not Python, it won't be able to unpickle
 205         # the exception so use level that won't be ignored by default.
 206         level = logging.ERROR
 207         # Even if the runtime is Python, the exception may be unpicklable if
 208         # it requires importing a class blocked by the sandbox so just send
 209         # back the exception representation.
 210         e = RuntimeError(repr(e))
 211       # While not strictly necessary for ApplicationError, do this to limit
 212       # differences with remote_api:handler.py.
 213       response.set_exception(pickle.dumps(e))
 214       logging.log(level, 'Exception while handling %s\n%s', request,
 215                   traceback.format_exc())
 216     encoded_response = response.Encode()
 217     logging.debug('Handled %s.%s in %0.4f',
 218                   request.service_name(),
 219                   request.method(),
 220                   time.time() - start_time)
 221     return [encoded_response]
 222
 223   def _handle_GET(self, environ, start_response):
 224     params = urlparse.parse_qs(environ['QUERY_STRING'])
 225     rtok = params.get('rtok', ['0'])[0]
 226
 227     start_response('200 OK', [('Content-Type', 'text/plain')])
 228     return [yaml.dump({'app_id': self._app_id,
 229                        'rtok': rtok})]
 230
 231   def __call__(self, environ, start_response):
 232     if environ['REQUEST_METHOD'] == 'GET':
 233       return self._handle_GET(environ, start_response)
 234     elif environ['REQUEST_METHOD'] == 'POST':
 235       return self._handle_POST(environ, start_response)
 236     else:
 237       start_response('405 Method Not Allowed', [])
 238       return []
 239
 240
 241 def setup_stubs(
 242     request_data,
 243     app_id,
 244     application_root,
 245     trusted,
 246     appidentity_email_address,
 247     appidentity_private_key_path,
 248     blobstore_path,
 249     datastore_consistency,
 250     datastore_path,
 251     datastore_require_indexes,
 252     datastore_auto_id_policy,
 253     images_host_prefix,
 254     logs_path,
 255     mail_smtp_host,
 256     mail_smtp_port,
 257     mail_smtp_user,
 258     mail_smtp_password,
 259     mail_enable_sendmail,
 260     mail_show_mail_body,
 261     matcher_prospective_search_path,
 262     search_index_path,
 263     taskqueue_auto_run_tasks,
 264     taskqueue_default_http_server,
 265     user_login_url,
 266     user_logout_url,
 267     default_gcs_bucket_name):
 268   """Configures the APIs hosted by this server.
 269
 270   Args:
 271     request_data: An apiproxy_stub.RequestInformation instance used by the
 272         stubs to lookup information about the request associated with an API
 273         call.
 274     app_id: The str application id e.g. "guestbook".
 275     application_root: The path to the directory containing the user's
 276         application e.g. "/home/joe/myapp".
 277     trusted: A bool indicating if privileged APIs should be made available.
 278     appidentity_email_address: Email address associated with a service account
 279         that has a downloadable key. May be None for no local application
 280         identity.
 281     appidentity_private_key_path: Path to private key file associated with
 282         service account (.pem format). Must be set if appidentity_email_address
 283         is set.
 284     blobstore_path: The path to the file that should be used for blobstore
 285         storage.
 286     datastore_consistency: The datastore_stub_util.BaseConsistencyPolicy to
 287         use as the datastore consistency policy.
 288     datastore_path: The path to the file that should be used for datastore
 289         storage.
 290     datastore_require_indexes: A bool indicating if the same production
 291         datastore indexes requirements should be enforced i.e. if True then
 292         a google.appengine.ext.db.NeedIndexError will be be raised if a query
 293         is executed without the required indexes.
 294     datastore_auto_id_policy: The type of sequence from which the datastore
 295         stub assigns auto IDs, either datastore_stub_util.SEQUENTIAL or
 296         datastore_stub_util.SCATTERED.
 297     images_host_prefix: The URL prefix (protocol://host:port) to prepend to
 298         image urls on calls to images.GetUrlBase.
 299     logs_path: Path to the file to store the logs data in.
 300     mail_smtp_host: The SMTP hostname that should be used when sending e-mails.
 301         If None then the mail_enable_sendmail argument is considered.
 302     mail_smtp_port: The SMTP port number that should be used when sending
 303         e-mails. If this value is None then mail_smtp_host must also be None.
 304     mail_smtp_user: The username to use when authenticating with the
 305         SMTP server. This value may be None if mail_smtp_host is also None or if
 306         the SMTP server does not require authentication.
 307     mail_smtp_password: The password to use when authenticating with the
 308         SMTP server. This value may be None if mail_smtp_host or mail_smtp_user
 309         is also None.
 310     mail_enable_sendmail: A bool indicating if sendmail should be used when
 311         sending e-mails. This argument is ignored if mail_smtp_host is not None.
 312     mail_show_mail_body: A bool indicating whether the body of sent e-mails
 313         should be written to the logs.
 314     matcher_prospective_search_path: The path to the file that should be used to
 315         save prospective search subscriptions.
 316     search_index_path: The path to the file that should be used for search index
 317         storage.
 318     taskqueue_auto_run_tasks: A bool indicating whether taskqueue tasks should
 319         be run automatically or it the must be manually triggered.
 320     taskqueue_default_http_server: A str containing the address of the http
 321         server that should be used to execute tasks.
 322     user_login_url: A str containing the url that should be used for user login.
 323     user_logout_url: A str containing the url that should be used for user
 324         logout.
 325     default_gcs_bucket_name: A str, overriding the default bucket behavior.
 326   """
 327
 328   identity_stub = app_identity_stub.AppIdentityServiceStub.Create(
 329       email_address=appidentity_email_address,
 330       private_key_path=appidentity_private_key_path)
 331   if default_gcs_bucket_name is not None:
 332     identity_stub.SetDefaultGcsBucketName(default_gcs_bucket_name)
 333   apiproxy_stub_map.apiproxy.RegisterStub('app_identity_service', identity_stub)
 334
 335   blob_storage = file_blob_storage.FileBlobStorage(blobstore_path, app_id)
 336   apiproxy_stub_map.apiproxy.RegisterStub(
 337       'blobstore',
 338       blobstore_stub.BlobstoreServiceStub(blob_storage,
 339                                           request_data=request_data))
 340
 341   apiproxy_stub_map.apiproxy.RegisterStub(
 342       'capability_service',
 343       capability_stub.CapabilityServiceStub())
 344
 345   apiproxy_stub_map.apiproxy.RegisterStub(
 346       'channel',
 347       channel_service_stub.ChannelServiceStub(request_data=request_data))
 348
 349   datastore_stub = datastore_sqlite_stub.DatastoreSqliteStub(
 350       app_id,
 351       datastore_path,
 352       datastore_require_indexes,
 353       trusted,
 354       root_path=application_root,
 355       auto_id_policy=datastore_auto_id_policy)
 356
 357   datastore_stub.SetConsistencyPolicy(datastore_consistency)
 358
 359   apiproxy_stub_map.apiproxy.ReplaceStub(
 360       'datastore_v3', datastore_stub)
 361
 362   apiproxy_stub_map.apiproxy.RegisterStub(
 363       'datastore_v4',
 364       datastore_v4_stub.DatastoreV4Stub(app_id))
 365
 366   apiproxy_stub_map.apiproxy.RegisterStub(
 367       'file',
 368       file_service_stub.FileServiceStub(blob_storage))
 369
 370   try:
 371     from google.appengine.api.images import images_stub
 372   except ImportError:
 373
 374     logging.warning('Could not initialize images API; you are likely missing '
 375                     'the Python "PIL" module.')
 376     # We register a stub which throws a NotImplementedError for most RPCs.
 377     from google.appengine.api.images import images_not_implemented_stub
 378     apiproxy_stub_map.apiproxy.RegisterStub(
 379         'images',
 380         images_not_implemented_stub.ImagesNotImplementedServiceStub(
 381             host_prefix=images_host_prefix))
 382   else:
 383     apiproxy_stub_map.apiproxy.RegisterStub(
 384         'images',
 385         images_stub.ImagesServiceStub(host_prefix=images_host_prefix))
 386
 387   apiproxy_stub_map.apiproxy.RegisterStub(
 388       'logservice',
 389       logservice_stub.LogServiceStub(logs_path=logs_path))
 390
 391   apiproxy_stub_map.apiproxy.RegisterStub(
 392       'mail',
 393       mail_stub.MailServiceStub(mail_smtp_host,
 394                                 mail_smtp_port,
 395                                 mail_smtp_user,
 396                                 mail_smtp_password,
 397                                 enable_sendmail=mail_enable_sendmail,
 398                                 show_mail_body=mail_show_mail_body))
 399
 400   apiproxy_stub_map.apiproxy.RegisterStub(
 401       'memcache',
 402       memcache_stub.MemcacheServiceStub())
 403
 404   apiproxy_stub_map.apiproxy.RegisterStub(
 405       'search',
 406       simple_search_stub.SearchServiceStub(index_file=search_index_path))
 407
 408   apiproxy_stub_map.apiproxy.RegisterStub(
 409       'modules',
 410       modules_stub.ModulesServiceStub(request_data))
 411
 412   apiproxy_stub_map.apiproxy.RegisterStub(
 413       'system',
 414       system_stub.SystemServiceStub(request_data=request_data))
 415
 416   apiproxy_stub_map.apiproxy.RegisterStub(
 417       'taskqueue',
 418       taskqueue_stub.TaskQueueServiceStub(
 419           root_path=application_root,
 420           auto_task_running=taskqueue_auto_run_tasks,
 421           default_http_server=taskqueue_default_http_server,
 422           request_data=request_data))
 423   apiproxy_stub_map.apiproxy.GetStub('taskqueue').StartBackgroundExecution()
 424
 425   apiproxy_stub_map.apiproxy.RegisterStub(
 426       'urlfetch',
 427       urlfetch_stub.URLFetchServiceStub())
 428
 429   apiproxy_stub_map.apiproxy.RegisterStub(
 430       'user',
 431       user_service_stub.UserServiceStub(login_url=user_login_url,
 432                                         logout_url=user_logout_url,
 433                                         request_data=request_data))
 434
 435   apiproxy_stub_map.apiproxy.RegisterStub(
 436       'xmpp',
 437       xmpp_service_stub.XmppServiceStub())
 438
 439   apiproxy_stub_map.apiproxy.RegisterStub(
 440       'matcher',
 441       prospective_search_stub.ProspectiveSearchStub(
 442           matcher_prospective_search_path,
 443           apiproxy_stub_map.apiproxy.GetStub('taskqueue')))
 444
 445   apiproxy_stub_map.apiproxy.RegisterStub(
 446       'remote_socket',
 447       _remote_socket_stub.RemoteSocketServiceStub())
 448
 449
 450 def maybe_convert_datastore_file_stub_data_to_sqlite(app_id, filename):
 451   if not os.access(filename, os.R_OK | os.W_OK):
 452     return
 453   try:
 454     with open(filename, 'rb') as f:
 455       if f.read(16) == 'SQLite format 3\x00':
 456         return
 457   except (IOError, OSError):
 458     return
 459   try:
 460     _convert_datastore_file_stub_data_to_sqlite(app_id, filename)
 461   except:
 462     logging.exception('Failed to convert datastore file stub data to sqlite.')
 463     raise
 464
 465
 466 def _convert_datastore_file_stub_data_to_sqlite(app_id, datastore_path):
 467   logging.info('Converting datastore stub data to sqlite.')
 468   previous_stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3')
 469   try:
 470     apiproxy_stub_map.apiproxy = apiproxy_stub_map.APIProxyStubMap()
 471     datastore_stub = datastore_file_stub.DatastoreFileStub(
 472         app_id, datastore_path, trusted=True, save_changes=False)
 473     apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', datastore_stub)
 474
 475     entities = _fetch_all_datastore_entities()
 476     sqlite_datastore_stub = datastore_sqlite_stub.DatastoreSqliteStub(
 477         app_id, datastore_path + '.sqlite', trusted=True)
 478     apiproxy_stub_map.apiproxy.ReplaceStub('datastore_v3',
 479                                            sqlite_datastore_stub)
 480     datastore.Put(entities)
 481     sqlite_datastore_stub.Close()
 482   finally:
 483     apiproxy_stub_map.apiproxy.ReplaceStub('datastore_v3', previous_stub)
 484
 485   shutil.copy(datastore_path, datastore_path + '.filestub')
 486   os.remove(datastore_path)
 487   shutil.move(datastore_path + '.sqlite', datastore_path)
 488   logging.info('Datastore conversion complete. File stub data has been backed '
 489                'up in %s', datastore_path + '.filestub')
 490
 491
 492 def _fetch_all_datastore_entities():
 493   """Returns all datastore entities from all namespaces as a list."""
 494   all_entities = []
 495   for namespace in datastore.Query('__namespace__').Run():
 496     namespace_name = namespace.key().name()
 497     for kind in datastore.Query('__kind__', namespace=namespace_name).Run():
 498       all_entities.extend(
 499           datastore.Query(kind.key().name(), namespace=namespace_name).Run())
 500   return all_entities
 501
 502
 503 def test_setup_stubs(
 504     request_data=None,
 505     app_id='myapp',
 506     application_root='/tmp/root',
 507     trusted=False,
 508     appidentity_email_address=None,
 509     appidentity_private_key_path=None,
 510     blobstore_path='/dev/null',
 511     datastore_consistency=None,
 512     datastore_path=':memory:',
 513     datastore_require_indexes=False,
 514     datastore_auto_id_policy=datastore_stub_util.SCATTERED,
 515     images_host_prefix='http://localhost:8080',
 516     logs_path=':memory:',
 517     mail_smtp_host='',
 518     mail_smtp_port=25,
 519     mail_smtp_user='',
 520     mail_smtp_password='',
 521     mail_enable_sendmail=False,
 522     mail_show_mail_body=False,
 523     matcher_prospective_search_path='/dev/null',
 524     search_index_path=None,
 525     taskqueue_auto_run_tasks=False,
 526     taskqueue_default_http_server='http://localhost:8080',
 527     user_login_url='/_ah/login?continue=%s',
 528     user_logout_url='/_ah/login?continue=%s',
 529     default_gcs_bucket_name=None):
 530   """Similar to setup_stubs with reasonable test defaults and recallable."""
 531
 532   # Reset the stub map between requests because a stub map only allows a
 533   # stub to be added once.
 534   apiproxy_stub_map.apiproxy = apiproxy_stub_map.APIProxyStubMap()
 535
 536   if datastore_consistency is None:
 537     datastore_consistency = (
 538         datastore_stub_util.PseudoRandomHRConsistencyPolicy())
 539
 540   setup_stubs(request_data,
 541               app_id,
 542               application_root,
 543               trusted,
 544               appidentity_email_address,
 545               appidentity_private_key_path,
 546               blobstore_path,
 547               datastore_consistency,
 548               datastore_path,
 549               datastore_require_indexes,
 550               datastore_auto_id_policy,
 551               images_host_prefix,
 552               logs_path,
 553               mail_smtp_host,
 554               mail_smtp_port,
 555               mail_smtp_user,
 556               mail_smtp_password,
 557               mail_enable_sendmail,
 558               mail_show_mail_body,
 559               matcher_prospective_search_path,
 560               search_index_path,
 561               taskqueue_auto_run_tasks,
 562               taskqueue_default_http_server,
 563               user_login_url,
 564               user_logout_url,
 565               default_gcs_bucket_name)
 566
 567
 568 def cleanup_stubs():
 569   """Do any necessary stub cleanup e.g. saving data."""
 570   # Saving datastore
 571   logging.info('Applying all pending transactions and saving the datastore')
 572   datastore_stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3')
 573   datastore_stub.Write()
 574   logging.info('Saving search indexes')
 575   apiproxy_stub_map.apiproxy.GetStub('search').Write()
 576   apiproxy_stub_map.apiproxy.GetStub('taskqueue').Shutdown()