python/google/appengine/tools/devappserver2/api_server.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2007 Google Inc.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #
  17 """Serves the stub App Engine APIs (e.g. memcache, datastore) over HTTP.
  18
  19 The Remote API protocol is used for communication.
  20 """
  21
  22
  23 import logging
  24 import os
  25 import pickle
  26 import shutil
  27 import socket
  28 import sys
  29 import tempfile
  30 import threading
  31 import time
  32 import traceback
  33 import urllib2
  34 import urlparse
  35
  36 import google
  37 import yaml
  38
  39 # Stubs
  40 from google.appengine.api import datastore_file_stub
  41 from google.appengine.api import mail_stub
  42 from google.appengine.api import urlfetch_stub
  43 from google.appengine.api import user_service_stub
  44 from google.appengine.api.app_identity import app_identity_stub
  45 from google.appengine.api.blobstore import blobstore_stub
  46 from google.appengine.api.blobstore import file_blob_storage
  47 from google.appengine.api.capabilities import capability_stub
  48 from google.appengine.api.channel import channel_service_stub
  49 from google.appengine.api.files import file_service_stub
  50 from google.appengine.api.logservice import logservice_stub
  51 from google.appengine.api.search import simple_search_stub
  52 from google.appengine.api.taskqueue import taskqueue_stub
  53 from google.appengine.api.prospective_search import prospective_search_stub
  54 from google.appengine.api.memcache import memcache_stub
  55 from google.appengine.api.modules import modules_stub
  56 from google.appengine.api.remote_socket import _remote_socket_stub
  57 from google.appengine.api.system import system_stub
  58 from google.appengine.api.xmpp import xmpp_service_stub
  59 from google.appengine.datastore import datastore_sqlite_stub
  60 from google.appengine.datastore import datastore_stub_util
  61 from google.appengine.datastore import datastore_v4_stub
  62
  63 from google.appengine.api import apiproxy_stub_map
  64 from google.appengine.api import datastore
  65 from google.appengine.ext.remote_api import remote_api_pb
  66 from google.appengine.ext.remote_api import remote_api_services
  67 from google.appengine.runtime import apiproxy_errors
  68 from google.appengine.tools.devappserver2 import wsgi_server
  69
  70
  71 # TODO: Remove this lock when stubs have been audited for thread
  72 # safety.
  73 GLOBAL_API_LOCK = threading.RLock()
  74
  75
  76 def _execute_request(request):
  77   """Executes an API method call and returns the response object.
  78
  79   Args:
  80     request: A remote_api_pb.Request object representing the API call e.g. a
  81         call to memcache.Get.
  82
  83   Returns:
  84     A ProtocolBuffer.ProtocolMessage representing the API response e.g. a
  85     memcache_service_pb.MemcacheGetResponse.
  86
  87   Raises:
  88     apiproxy_errors.CallNotFoundError: if the requested method doesn't exist.
  89     apiproxy_errors.ApplicationError: if the API method calls fails.
  90   """
  91   service = request.service_name()
  92   method = request.method()
  93   if request.has_request_id():
  94     request_id = request.request_id()
  95   else:
  96     logging.error('Received a request without request_id: %s', request)
  97     request_id = None
  98
  99   service_methods = remote_api_services.SERVICE_PB_MAP.get(service, {})
 100   request_class, response_class = service_methods.get(method, (None, None))
 101   if not request_class:
 102     raise apiproxy_errors.CallNotFoundError('%s.%s does not exist' % (service,
 103                                                                       method))
 104
 105   request_data = request_class()
 106   request_data.ParseFromString(request.request())
 107   response_data = response_class()
 108   service_stub = apiproxy_stub_map.apiproxy.GetStub(service)
 109
 110   def make_request():
 111     service_stub.MakeSyncCall(service,
 112                               method,
 113                               request_data,
 114                               response_data,
 115                               request_id)
 116
 117   # If the service has not declared itself as threadsafe acquire
 118   # GLOBAL_API_LOCK.
 119   if service_stub.THREADSAFE:
 120     make_request()
 121   else:
 122     with GLOBAL_API_LOCK:
 123       make_request()
 124   return response_data
 125
 126
 127 class APIServer(wsgi_server.WsgiServer):
 128   """Serves API calls over HTTP."""
 129
 130   def __init__(self, host, port, app_id):
 131     self._app_id = app_id
 132     self._host = host
 133     super(APIServer, self).__init__((host, port), self)
 134
 135   def start(self):
 136     """Start the API Server."""
 137     super(APIServer, self).start()
 138     logging.info('Starting API server at: http://%s:%d', self._host, self.port)
 139
 140   def quit(self):
 141     cleanup_stubs()
 142     super(APIServer, self).quit()
 143
 144   def _handle_POST(self, environ, start_response):
 145     start_response('200 OK', [('Content-Type', 'application/octet-stream')])
 146
 147     start_time = time.time()
 148     response = remote_api_pb.Response()
 149     try:
 150       request = remote_api_pb.Request()
 151       # NOTE: Exceptions encountered when parsing the PB or handling the request
 152       # will be propagated back to the caller the same way as exceptions raised
 153       # by the actual API call.
 154       if environ.get('HTTP_TRANSFER_ENCODING') == 'chunked':
 155         # CherryPy concatenates all chunks  when 'wsgi.input' is read but v3.2.2
 156         # will not return even when all of the data in all chunks has been
 157         # read. See: https://bitbucket.org/cherrypy/cherrypy/issue/1131.
 158         wsgi_input = environ['wsgi.input'].read(2**32)
 159       else:
 160         wsgi_input = environ['wsgi.input'].read(int(environ['CONTENT_LENGTH']))
 161       request.ParseFromString(wsgi_input)
 162       api_response = _execute_request(request).Encode()
 163       response.set_response(api_response)
 164     except Exception, e:
 165       if isinstance(e, apiproxy_errors.ApplicationError):
 166         level = logging.DEBUG
 167         application_error = response.mutable_application_error()
 168         application_error.set_code(e.application_error)
 169         application_error.set_detail(e.error_detail)
 170       else:
 171         # If the runtime instance is not Python, it won't be able to unpickle
 172         # the exception so use level that won't be ignored by default.
 173         level = logging.ERROR
 174         # Even if the runtime is Python, the exception may be unpicklable if
 175         # it requires importing a class blocked by the sandbox so just send
 176         # back the exception representation.
 177         e = RuntimeError(repr(e))
 178       # While not strictly necessary for ApplicationError, do this to limit
 179       # differences with remote_api:handler.py.
 180       response.set_exception(pickle.dumps(e))
 181       logging.log(level, 'Exception while handling %s\n%s', request,
 182                   traceback.format_exc())
 183     encoded_response = response.Encode()
 184     logging.debug('Handled %s.%s in %0.4f',
 185                   request.service_name(),
 186                   request.method(),
 187                   time.time() - start_time)
 188     return [encoded_response]
 189
 190   def _handle_GET(self, environ, start_response):
 191     params = urlparse.parse_qs(environ['QUERY_STRING'])
 192     rtok = params.get('rtok', ['0'])[0]
 193
 194     start_response('200 OK', [('Content-Type', 'text/plain')])
 195     return [yaml.dump({'app_id': self._app_id,
 196                        'rtok': rtok})]
 197
 198   def __call__(self, environ, start_response):
 199     if environ['REQUEST_METHOD'] == 'GET':
 200       return self._handle_GET(environ, start_response)
 201     elif environ['REQUEST_METHOD'] == 'POST':
 202       return self._handle_POST(environ, start_response)
 203     else:
 204       start_response('405 Method Not Allowed', [])
 205       return []
 206
 207
 208 def setup_stubs(
 209     request_data,
 210     app_id,
 211     application_root,
 212     trusted,
 213     appidentity_email_address,
 214     appidentity_private_key_path,
 215     blobstore_path,
 216     datastore_consistency,
 217     datastore_path,
 218     datastore_require_indexes,
 219     datastore_auto_id_policy,
 220     images_host_prefix,
 221     logs_path,
 222     mail_smtp_host,
 223     mail_smtp_port,
 224     mail_smtp_user,
 225     mail_smtp_password,
 226     mail_enable_sendmail,
 227     mail_show_mail_body,
 228     matcher_prospective_search_path,
 229     search_index_path,
 230     taskqueue_auto_run_tasks,
 231     taskqueue_default_http_server,
 232     user_login_url,
 233     user_logout_url,
 234     default_gcs_bucket_name):
 235   """Configures the APIs hosted by this server.
 236
 237   Args:
 238     request_data: An apiproxy_stub.RequestInformation instance used by the
 239         stubs to lookup information about the request associated with an API
 240         call.
 241     app_id: The str application id e.g. "guestbook".
 242     application_root: The path to the directory containing the user's
 243         application e.g. "/home/joe/myapp".
 244     trusted: A bool indicating if privileged APIs should be made available.
 245     appidentity_email_address: Email address associated with a service account
 246         that has a downloadable key. May be None for no local application
 247         identity.
 248     appidentity_private_key_path: Path to private key file associated with
 249         service account (.pem format). Must be set if appidentity_email_address
 250         is set.
 251     blobstore_path: The path to the file that should be used for blobstore
 252         storage.
 253     datastore_consistency: The datastore_stub_util.BaseConsistencyPolicy to
 254         use as the datastore consistency policy.
 255     datastore_path: The path to the file that should be used for datastore
 256         storage.
 257     datastore_require_indexes: A bool indicating if the same production
 258         datastore indexes requirements should be enforced i.e. if True then
 259         a google.appengine.ext.db.NeedIndexError will be be raised if a query
 260         is executed without the required indexes.
 261     datastore_auto_id_policy: The type of sequence from which the datastore
 262         stub assigns auto IDs, either datastore_stub_util.SEQUENTIAL or
 263         datastore_stub_util.SCATTERED.
 264     images_host_prefix: The URL prefix (protocol://host:port) to prepend to
 265         image urls on calls to images.GetUrlBase.
 266     logs_path: Path to the file to store the logs data in.
 267     mail_smtp_host: The SMTP hostname that should be used when sending e-mails.
 268         If None then the mail_enable_sendmail argument is considered.
 269     mail_smtp_port: The SMTP port number that should be used when sending
 270         e-mails. If this value is None then mail_smtp_host must also be None.
 271     mail_smtp_user: The username to use when authenticating with the
 272         SMTP server. This value may be None if mail_smtp_host is also None or if
 273         the SMTP server does not require authentication.
 274     mail_smtp_password: The password to use when authenticating with the
 275         SMTP server. This value may be None if mail_smtp_host or mail_smtp_user
 276         is also None.
 277     mail_enable_sendmail: A bool indicating if sendmail should be used when
 278         sending e-mails. This argument is ignored if mail_smtp_host is not None.
 279     mail_show_mail_body: A bool indicating whether the body of sent e-mails
 280         should be written to the logs.
 281     matcher_prospective_search_path: The path to the file that should be used to
 282         save prospective search subscriptions.
 283     search_index_path: The path to the file that should be used for search index
 284         storage.
 285     taskqueue_auto_run_tasks: A bool indicating whether taskqueue tasks should
 286         be run automatically or it the must be manually triggered.
 287     taskqueue_default_http_server: A str containing the address of the http
 288         server that should be used to execute tasks.
 289     user_login_url: A str containing the url that should be used for user login.
 290     user_logout_url: A str containing the url that should be used for user
 291         logout.
 292     default_gcs_bucket_name: A str, overriding the default bucket behavior.
 293   """
 294
 295   identity_stub = app_identity_stub.AppIdentityServiceStub.Create(
 296       email_address=appidentity_email_address,
 297       private_key_path=appidentity_private_key_path)
 298   if default_gcs_bucket_name is not None:
 299     identity_stub.SetDefaultGcsBucketName(default_gcs_bucket_name)
 300   apiproxy_stub_map.apiproxy.RegisterStub('app_identity_service', identity_stub)
 301
 302   blob_storage = file_blob_storage.FileBlobStorage(blobstore_path, app_id)
 303   apiproxy_stub_map.apiproxy.RegisterStub(
 304       'blobstore',
 305       blobstore_stub.BlobstoreServiceStub(blob_storage,
 306                                           request_data=request_data))
 307
 308   apiproxy_stub_map.apiproxy.RegisterStub(
 309       'capability_service',
 310       capability_stub.CapabilityServiceStub())
 311
 312   apiproxy_stub_map.apiproxy.RegisterStub(
 313       'channel',
 314       channel_service_stub.ChannelServiceStub(request_data=request_data))
 315
 316   datastore_stub = datastore_sqlite_stub.DatastoreSqliteStub(
 317       app_id,
 318       datastore_path,
 319       datastore_require_indexes,
 320       trusted,
 321       root_path=application_root,
 322       auto_id_policy=datastore_auto_id_policy)
 323
 324   datastore_stub.SetConsistencyPolicy(datastore_consistency)
 325
 326   apiproxy_stub_map.apiproxy.ReplaceStub(
 327       'datastore_v3', datastore_stub)
 328
 329   apiproxy_stub_map.apiproxy.RegisterStub(
 330       'datastore_v4',
 331       datastore_v4_stub.DatastoreV4Stub(app_id))
 332
 333   apiproxy_stub_map.apiproxy.RegisterStub(
 334       'file',
 335       file_service_stub.FileServiceStub(blob_storage))
 336
 337   try:
 338     from google.appengine.api.images import images_stub
 339   except ImportError:
 340
 341     logging.warning('Could not initialize images API; you are likely missing '
 342                     'the Python "PIL" module.')
 343     # We register a stub which throws a NotImplementedError for most RPCs.
 344     from google.appengine.api.images import images_not_implemented_stub
 345     apiproxy_stub_map.apiproxy.RegisterStub(
 346         'images',
 347         images_not_implemented_stub.ImagesNotImplementedServiceStub(
 348             host_prefix=images_host_prefix))
 349   else:
 350     apiproxy_stub_map.apiproxy.RegisterStub(
 351         'images',
 352         images_stub.ImagesServiceStub(host_prefix=images_host_prefix))
 353
 354   apiproxy_stub_map.apiproxy.RegisterStub(
 355       'logservice',
 356       logservice_stub.LogServiceStub(logs_path=logs_path))
 357
 358   apiproxy_stub_map.apiproxy.RegisterStub(
 359       'mail',
 360       mail_stub.MailServiceStub(mail_smtp_host,
 361                                 mail_smtp_port,
 362                                 mail_smtp_user,
 363                                 mail_smtp_password,
 364                                 enable_sendmail=mail_enable_sendmail,
 365                                 show_mail_body=mail_show_mail_body))
 366
 367   apiproxy_stub_map.apiproxy.RegisterStub(
 368       'memcache',
 369       memcache_stub.MemcacheServiceStub())
 370
 371   apiproxy_stub_map.apiproxy.RegisterStub(
 372       'search',
 373       simple_search_stub.SearchServiceStub(index_file=search_index_path))
 374
 375   apiproxy_stub_map.apiproxy.RegisterStub(
 376       'modules',
 377       modules_stub.ModulesServiceStub(request_data))
 378
 379   apiproxy_stub_map.apiproxy.RegisterStub(
 380       'system',
 381       system_stub.SystemServiceStub(request_data=request_data))
 382
 383   apiproxy_stub_map.apiproxy.RegisterStub(
 384       'taskqueue',
 385       taskqueue_stub.TaskQueueServiceStub(
 386           root_path=application_root,
 387           auto_task_running=taskqueue_auto_run_tasks,
 388           default_http_server=taskqueue_default_http_server,
 389           request_data=request_data))
 390   apiproxy_stub_map.apiproxy.GetStub('taskqueue').StartBackgroundExecution()
 391
 392   apiproxy_stub_map.apiproxy.RegisterStub(
 393       'urlfetch',
 394       urlfetch_stub.URLFetchServiceStub())
 395
 396   apiproxy_stub_map.apiproxy.RegisterStub(
 397       'user',
 398       user_service_stub.UserServiceStub(login_url=user_login_url,
 399                                         logout_url=user_logout_url,
 400                                         request_data=request_data))
 401
 402   apiproxy_stub_map.apiproxy.RegisterStub(
 403       'xmpp',
 404       xmpp_service_stub.XmppServiceStub())
 405
 406   apiproxy_stub_map.apiproxy.RegisterStub(
 407       'matcher',
 408       prospective_search_stub.ProspectiveSearchStub(
 409           matcher_prospective_search_path,
 410           apiproxy_stub_map.apiproxy.GetStub('taskqueue')))
 411
 412   apiproxy_stub_map.apiproxy.RegisterStub(
 413       'remote_socket',
 414       _remote_socket_stub.RemoteSocketServiceStub())
 415
 416
 417 def maybe_convert_datastore_file_stub_data_to_sqlite(app_id, filename):
 418   if not os.access(filename, os.R_OK | os.W_OK):
 419     return
 420   try:
 421     with open(filename, 'rb') as f:
 422       if f.read(16) == 'SQLite format 3\x00':
 423         return
 424   except (IOError, OSError):
 425     return
 426   try:
 427     _convert_datastore_file_stub_data_to_sqlite(app_id, filename)
 428   except:
 429     logging.exception('Failed to convert datastore file stub data to sqlite.')
 430     raise
 431
 432
 433 def _convert_datastore_file_stub_data_to_sqlite(app_id, datastore_path):
 434   logging.info('Converting datastore stub data to sqlite.')
 435   previous_stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3')
 436   try:
 437     apiproxy_stub_map.apiproxy = apiproxy_stub_map.APIProxyStubMap()
 438     datastore_stub = datastore_file_stub.DatastoreFileStub(
 439         app_id, datastore_path, trusted=True, save_changes=False)
 440     apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', datastore_stub)
 441
 442     entities = _fetch_all_datastore_entities()
 443     sqlite_datastore_stub = datastore_sqlite_stub.DatastoreSqliteStub(
 444         app_id, datastore_path + '.sqlite', trusted=True)
 445     apiproxy_stub_map.apiproxy.ReplaceStub('datastore_v3',
 446                                            sqlite_datastore_stub)
 447     datastore.Put(entities)
 448     sqlite_datastore_stub.Close()
 449   finally:
 450     apiproxy_stub_map.apiproxy.ReplaceStub('datastore_v3', previous_stub)
 451
 452   shutil.copy(datastore_path, datastore_path + '.filestub')
 453   os.remove(datastore_path)
 454   shutil.move(datastore_path + '.sqlite', datastore_path)
 455   logging.info('Datastore conversion complete. File stub data has been backed '
 456                'up in %s', datastore_path + '.filestub')
 457
 458
 459 def _fetch_all_datastore_entities():
 460   """Returns all datastore entities from all namespaces as a list."""
 461   all_entities = []
 462   for namespace in datastore.Query('__namespace__').Run():
 463     namespace_name = namespace.key().name()
 464     for kind in datastore.Query('__kind__', namespace=namespace_name).Run():
 465       all_entities.extend(
 466           datastore.Query(kind.key().name(), namespace=namespace_name).Run())
 467   return all_entities
 468
 469
 470 def test_setup_stubs(
 471     request_data=None,
 472     app_id='myapp',
 473     application_root='/tmp/root',
 474     trusted=False,
 475     appidentity_email_address=None,
 476     appidentity_private_key_path=None,
 477     blobstore_path='/dev/null',
 478     datastore_consistency=None,
 479     datastore_path=':memory:',
 480     datastore_require_indexes=False,
 481     datastore_auto_id_policy=datastore_stub_util.SCATTERED,
 482     images_host_prefix='http://localhost:8080',
 483     logs_path=':memory:',
 484     mail_smtp_host='',
 485     mail_smtp_port=25,
 486     mail_smtp_user='',
 487     mail_smtp_password='',
 488     mail_enable_sendmail=False,
 489     mail_show_mail_body=False,
 490     matcher_prospective_search_path='/dev/null',
 491     search_index_path=None,
 492     taskqueue_auto_run_tasks=False,
 493     taskqueue_default_http_server='http://localhost:8080',
 494     user_login_url='/_ah/login?continue=%s',
 495     user_logout_url='/_ah/login?continue=%s',
 496     default_gcs_bucket_name=None):
 497   """Similar to setup_stubs with reasonable test defaults and recallable."""
 498
 499   # Reset the stub map between requests because a stub map only allows a
 500   # stub to be added once.
 501   apiproxy_stub_map.apiproxy = apiproxy_stub_map.APIProxyStubMap()
 502
 503   if datastore_consistency is None:
 504     datastore_consistency = (
 505         datastore_stub_util.PseudoRandomHRConsistencyPolicy())
 506
 507   setup_stubs(request_data,
 508               app_id,
 509               application_root,
 510               trusted,
 511               appidentity_email_address,
 512               appidentity_private_key_path,
 513               blobstore_path,
 514               datastore_consistency,
 515               datastore_path,
 516               datastore_require_indexes,
 517               datastore_auto_id_policy,
 518               images_host_prefix,
 519               logs_path,
 520               mail_smtp_host,
 521               mail_smtp_port,
 522               mail_smtp_user,
 523               mail_smtp_password,
 524               mail_enable_sendmail,
 525               mail_show_mail_body,
 526               matcher_prospective_search_path,
 527               search_index_path,
 528               taskqueue_auto_run_tasks,
 529               taskqueue_default_http_server,
 530               user_login_url,
 531               user_logout_url,
 532               default_gcs_bucket_name)
 533
 534
 535 def cleanup_stubs():
 536   """Do any necessary stub cleanup e.g. saving data."""
 537   # Saving datastore
 538   logging.info('Applying all pending transactions and saving the datastore')
 539   datastore_stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3')
 540   datastore_stub.Write()
 541   logging.info('Saving search indexes')
 542   apiproxy_stub_map.apiproxy.GetStub('search').Write()
 543   apiproxy_stub_map.apiproxy.GetStub('taskqueue').Shutdown()