python/google/appengine/tools/api_server.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2007 Google Inc.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #
  17 """Serves the stub App Engine APIs (e.g. memcache, datastore) over HTTP.
  18
  19 The Remote API protocol is used for communication.
  20 """
  21
  22 from __future__ import with_statement
  23
  24
  25 import BaseHTTPServer
  26 import httplib
  27 import logging
  28 import os.path
  29 import pickle
  30 import socket
  31 import SocketServer
  32 import subprocess
  33 import sys
  34 import tempfile
  35 import threading
  36 import time
  37 import traceback
  38 import urllib2
  39 import urlparse
  40 import wsgiref.headers
  41
  42 import google
  43 import yaml
  44
  45
  46 from google.appengine.api import mail_stub
  47 from google.appengine.api import request_info
  48 from google.appengine.api import urlfetch_stub
  49 from google.appengine.api import user_service_stub
  50 from google.appengine.api.app_identity import app_identity_stub
  51 from google.appengine.api.blobstore import blobstore_stub
  52 from google.appengine.api.blobstore import file_blob_storage
  53 from google.appengine.api.capabilities import capability_stub
  54 from google.appengine.api.channel import channel_service_stub
  55 from google.appengine.api.files import file_service_stub
  56 from google.appengine.api.logservice import logservice_stub
  57 from google.appengine.api.search import simple_search_stub
  58 from google.appengine.api.taskqueue import taskqueue_stub
  59 from google.appengine.api.prospective_search import prospective_search_stub
  60 from google.appengine.api.memcache import memcache_stub
  61 from google.appengine.api.system import system_stub
  62 from google.appengine.api.xmpp import xmpp_service_stub
  63 from google.appengine.api import datastore_file_stub
  64 from google.appengine.datastore import datastore_sqlite_stub
  65 from google.appengine.datastore import datastore_stub_util
  66 from google.appengine.datastore import datastore_v4_stub
  67
  68 from google.appengine.api import apiproxy_stub_map
  69 from google.appengine.ext.remote_api import remote_api_pb
  70 from google.appengine.ext.remote_api import remote_api_services
  71 from google.appengine.runtime import apiproxy_errors
  72
  73
  74 QUIT_PATH = '/quit'
  75
  76
  77 GLOBAL_API_LOCK = threading.RLock()
  78
  79
  80 class Error(Exception):
  81   pass
  82
  83
  84 def _ClearDatastoreStorage(datastore_path):
  85   """Delete the datastore storage file at the given path."""
  86
  87   if os.path.lexists(datastore_path):
  88     try:
  89       os.remove(datastore_path)
  90     except OSError, e:
  91       logging.warning('Failed to remove datastore file %r: %s',
  92                       datastore_path,
  93                       e)
  94
  95
  96 def _ClearProspectiveSearchStorage(prospective_search_path):
  97   """Delete the perspective search storage file at the given path."""
  98
  99   if os.path.lexists(prospective_search_path):
 100     try:
 101       os.remove(prospective_search_path)
 102     except OSError, e:
 103       logging.warning('Failed to remove prospective search file %r: %s',
 104                       prospective_search_path,
 105                       e)
 106
 107
 108
 109
 110 THREAD_SAFE_SERVICES = frozenset((
 111     'app_identity_service',
 112     'capability_service',
 113     'channel',
 114     'logservice',
 115     'mail',
 116     'memcache',
 117     'remote_socket',
 118     'urlfetch',
 119     'user',
 120     'xmpp',
 121 ))
 122
 123
 124 def _ExecuteRequest(request):
 125   """Executes an API method call and returns the response object.
 126
 127   Args:
 128     request: A remote_api.Request object representing the API call e.g. a call
 129         to memcache.Get.
 130
 131   Returns:
 132     A ProtocolBuffer.ProtocolMessage representing the API response e.g. a
 133     memcache_service_pb.MemcacheGetResponse.
 134
 135   Raises:
 136     apiproxy_errors.CallNotFoundError: if the requested method doesn't exist.
 137     apiproxy_errors.ApplicationError: if the API method calls fails.
 138   """
 139   service = request.service_name()
 140   method = request.method()
 141   service_methods = remote_api_services.SERVICE_PB_MAP.get(service, {})
 142   request_class, response_class = service_methods.get(method, (None, None))
 143   if not request_class:
 144     raise apiproxy_errors.CallNotFoundError('%s.%s does not exist' % (service,
 145                                                                       method))
 146
 147   request_data = request_class()
 148   request_data.ParseFromString(request.request())
 149   response_data = response_class()
 150
 151   def MakeRequest():
 152     apiproxy_stub_map.MakeSyncCall(service, method, request_data,
 153                                    response_data)
 154
 155
 156
 157   if service in THREAD_SAFE_SERVICES:
 158     MakeRequest()
 159   else:
 160     with GLOBAL_API_LOCK:
 161       MakeRequest()
 162   return response_data
 163
 164
 165 class APIRequestHandler(BaseHTTPServer.BaseHTTPRequestHandler):
 166   """Handler for all API server HTTP requests."""
 167
 168   def log_message(self, format, *args):
 169     logging.debug(format, *args)
 170
 171   def do_GET(self):
 172     if self.path == QUIT_PATH:
 173       self._HandleShutdown()
 174     else:
 175       params = urlparse.parse_qs(urlparse.urlparse(self.path).query)
 176       rtok = params.get('rtok', ['0'])[0]
 177
 178       self.send_response(httplib.OK)
 179       self.send_header('Content-Type', 'text/plain')
 180       self.end_headers()
 181       self.wfile.write(yaml.dump({
 182           'app_id': self.server.app_id,
 183           'rtok': rtok,
 184           }))
 185
 186   def _HandleShutdown(self):
 187     """Handles a request for the API Server to exit."""
 188     self.send_response(httplib.OK)
 189     self.send_header('Content-Type', 'text/plain')
 190     self.end_headers()
 191     self.wfile.write('API Server Quitting')
 192     self.server.shutdown()
 193
 194   def do_POST(self):
 195     """Handles a single API request e.g. memcache.Get()."""
 196     self.send_response(httplib.OK)
 197     self.send_header('Content-Type', 'application/octet-stream')
 198     self.end_headers()
 199
 200     response = remote_api_pb.Response()
 201     try:
 202       request = remote_api_pb.Request()
 203
 204
 205
 206       request.ParseFromString(
 207           self.rfile.read(int(self.headers['content-length'])))
 208       api_response = _ExecuteRequest(request).Encode()
 209       response.set_response(api_response)
 210     except Exception, e:
 211       logging.debug('Exception while handling %s\n%s',
 212                     request,
 213                     traceback.format_exc())
 214       response.set_exception(pickle.dumps(e))
 215       if isinstance(e, apiproxy_errors.ApplicationError):
 216         application_error = response.mutable_application_error()
 217         application_error.set_code(e.application_error)
 218         application_error.set_detail(e.error_detail)
 219     self.wfile.write(response.Encode())
 220
 221
 222 class APIServer(SocketServer.ThreadingMixIn, BaseHTTPServer.HTTPServer):
 223   """Serves API calls over HTTP."""
 224
 225   def __init__(self, server_address, app_id):
 226     BaseHTTPServer.HTTPServer.__init__(self, server_address, APIRequestHandler)
 227     self.app_id = app_id
 228
 229
 230 def _SetupStubs(
 231     app_id,
 232     application_root,
 233     appidentity_email_address,
 234     appidentity_private_key_path,
 235     trusted,
 236     blobstore_path,
 237     use_sqlite,
 238     auto_id_policy,
 239     high_replication,
 240     datastore_path,
 241     datastore_require_indexes,
 242     images_host_prefix,
 243     logs_path,
 244     mail_smtp_host,
 245     mail_smtp_port,
 246     mail_smtp_user,
 247     mail_smtp_password,
 248     mail_enable_sendmail,
 249     mail_show_mail_body,
 250     mail_allow_tls,
 251     matcher_prospective_search_path,
 252     taskqueue_auto_run_tasks,
 253     taskqueue_task_retry_seconds,
 254     taskqueue_default_http_server,
 255     user_login_url,
 256     user_logout_url,
 257     default_gcs_bucket_name):
 258   """Configures the APIs hosted by this server.
 259
 260   Args:
 261     app_id: The str application id e.g. "guestbook".
 262     application_root: The path to the directory containing the user's
 263         application e.g. "/home/bquinlan/myapp".
 264     trusted: A bool indicating if privileged APIs should be made available.
 265     blobstore_path: The path to the file that should be used for blobstore
 266         storage.
 267     use_sqlite: A bool indicating whether DatastoreSqliteStub or
 268         DatastoreFileStub should be used.
 269     auto_id_policy: One of datastore_stub_util.SEQUENTIAL or .SCATTERED,
 270         indicating whether the Datastore stub should assign IDs sequentially
 271         or scattered.
 272     high_replication: A bool indicating whether to use the high replication
 273         consistency model.
 274     datastore_path: The path to the file that should be used for datastore
 275         storage.
 276     datastore_require_indexes: A bool indicating if the same production
 277         datastore indexes requirements should be enforced i.e. if True then
 278         a google.appengine.ext.db.NeedIndexError will be be raised if a query
 279         is executed without the required indexes.
 280     images_host_prefix: The URL prefix (protocol://host:port) to preprend to
 281         image urls on calls to images.GetUrlBase.
 282     logs_path: Path to the file to store the logs data in.
 283     mail_smtp_host: The SMTP hostname that should be used when sending e-mails.
 284         If None then the mail_enable_sendmail argument is considered.
 285     mail_smtp_port: The SMTP port number that should be used when sending
 286         e-mails. If this value is None then mail_smtp_host must also be None.
 287     mail_smtp_user: The username to use when authenticating with the
 288         SMTP server. This value may be None if mail_smtp_host is also None or if
 289         the SMTP server does not require authentication.
 290     mail_smtp_password: The password to use when authenticating with the
 291         SMTP server. This value may be None if mail_smtp_host or mail_smtp_user
 292         is also None.
 293     mail_enable_sendmail: A bool indicating if sendmail should be used when
 294         sending e-mails. This argument is ignored if mail_smtp_host is not None.
 295     mail_show_mail_body: A bool indicating whether the body of sent e-mails
 296         should be written to the logs.
 297     mail_allow_tls: A bool indicating whether to allow TLS support.
 298     matcher_prospective_search_path: The path to the file that should be used to
 299         save prospective search subscriptions.
 300     taskqueue_auto_run_tasks: A bool indicating whether taskqueue tasks should
 301         be run automatically or it the must be manually triggered.
 302     taskqueue_task_retry_seconds: An int representing the number of seconds to
 303         wait before a retrying a failed taskqueue task.
 304     taskqueue_default_http_server: A str containing the address of the http
 305         server that should be used to execute tasks.
 306     user_login_url: A str containing the url that should be used for user login.
 307     user_logout_url: A str containing the url that should be used for user
 308         logout.
 309     default_gcs_bucket_name: A str overriding the usual default bucket name.
 310   """
 311
 312
 313
 314
 315
 316   os.environ['APPLICATION_ID'] = app_id
 317
 318
 319
 320   tmp_app_identity_stub = app_identity_stub.AppIdentityServiceStub.Create(
 321       email_address=appidentity_email_address,
 322       private_key_path=appidentity_private_key_path)
 323   if default_gcs_bucket_name is not None:
 324     tmp_app_identity_stub.SetDefaultGcsBucketName(default_gcs_bucket_name)
 325   apiproxy_stub_map.apiproxy.RegisterStub(
 326       'app_identity_service', tmp_app_identity_stub)
 327
 328   blob_storage = file_blob_storage.FileBlobStorage(blobstore_path, app_id)
 329   apiproxy_stub_map.apiproxy.RegisterStub(
 330       'blobstore',
 331       blobstore_stub.BlobstoreServiceStub(blob_storage))
 332
 333   apiproxy_stub_map.apiproxy.RegisterStub(
 334       'capability_service',
 335       capability_stub.CapabilityServiceStub())
 336
 337
 338
 339
 340
 341
 342
 343
 344   apiproxy_stub_map.apiproxy.RegisterStub(
 345       'channel',
 346       channel_service_stub.ChannelServiceStub())
 347
 348   if use_sqlite:
 349     datastore = datastore_sqlite_stub.DatastoreSqliteStub(
 350         app_id,
 351         datastore_path,
 352         datastore_require_indexes,
 353         trusted,
 354         root_path=application_root,
 355         auto_id_policy=auto_id_policy)
 356   else:
 357     datastore = datastore_file_stub.DatastoreFileStub(
 358         app_id,
 359         datastore_path,
 360         datastore_require_indexes,
 361         trusted,
 362         root_path=application_root,
 363         auto_id_policy=auto_id_policy)
 364
 365   if high_replication:
 366     datastore.SetConsistencyPolicy(
 367         datastore_stub_util.TimeBasedHRConsistencyPolicy())
 368
 369   apiproxy_stub_map.apiproxy.RegisterStub(
 370       'datastore_v3', datastore)
 371
 372   apiproxy_stub_map.apiproxy.RegisterStub(
 373       'datastore_v4',
 374       datastore_v4_stub.DatastoreV4Stub(app_id))
 375
 376   apiproxy_stub_map.apiproxy.RegisterStub(
 377       'file',
 378       file_service_stub.FileServiceStub(blob_storage))
 379
 380   try:
 381     from google.appengine.api.images import images_stub
 382   except ImportError:
 383
 384
 385     logging.warning('Could not initialize images API; you are likely missing '
 386                     'the Python "PIL" module.')
 387
 388     from google.appengine.api.images import images_not_implemented_stub
 389     apiproxy_stub_map.apiproxy.RegisterStub(
 390         'images',
 391         images_not_implemented_stub.ImagesNotImplementedServiceStub())
 392   else:
 393     apiproxy_stub_map.apiproxy.RegisterStub(
 394         'images',
 395         images_stub.ImagesServiceStub(host_prefix=images_host_prefix))
 396
 397   apiproxy_stub_map.apiproxy.RegisterStub(
 398       'logservice',
 399       logservice_stub.LogServiceStub(logs_path=logs_path))
 400
 401   apiproxy_stub_map.apiproxy.RegisterStub(
 402       'mail',
 403       mail_stub.MailServiceStub(mail_smtp_host,
 404                                 mail_smtp_port,
 405                                 mail_smtp_user,
 406                                 mail_smtp_password,
 407                                 enable_sendmail=mail_enable_sendmail,
 408                                 show_mail_body=mail_show_mail_body,
 409                                 allow_tls=mail_allow_tls))
 410
 411   apiproxy_stub_map.apiproxy.RegisterStub(
 412       'memcache',
 413       memcache_stub.MemcacheServiceStub())
 414
 415   apiproxy_stub_map.apiproxy.RegisterStub(
 416       'search',
 417       simple_search_stub.SearchServiceStub())
 418
 419   apiproxy_stub_map.apiproxy.RegisterStub('system',
 420                                           system_stub.SystemServiceStub())
 421
 422   apiproxy_stub_map.apiproxy.RegisterStub(
 423       'taskqueue',
 424       taskqueue_stub.TaskQueueServiceStub(
 425           root_path=application_root,
 426           auto_task_running=taskqueue_auto_run_tasks,
 427           task_retry_seconds=taskqueue_task_retry_seconds,
 428           default_http_server=taskqueue_default_http_server))
 429   apiproxy_stub_map.apiproxy.GetStub('taskqueue').StartBackgroundExecution()
 430
 431   apiproxy_stub_map.apiproxy.RegisterStub(
 432       'urlfetch',
 433       urlfetch_stub.URLFetchServiceStub())
 434
 435   apiproxy_stub_map.apiproxy.RegisterStub(
 436       'user',
 437       user_service_stub.UserServiceStub(login_url=user_login_url,
 438                                         logout_url=user_logout_url))
 439
 440   apiproxy_stub_map.apiproxy.RegisterStub(
 441       'xmpp',
 442       xmpp_service_stub.XmppServiceStub())
 443
 444   apiproxy_stub_map.apiproxy.RegisterStub(
 445       'matcher',
 446       prospective_search_stub.ProspectiveSearchStub(
 447           matcher_prospective_search_path,
 448           apiproxy_stub_map.apiproxy.GetStub('taskqueue')))
 449
 450
 451 def _TearDownStubs():
 452   """Clean up any stubs that need cleanup."""
 453
 454   logging.info('Applying all pending transactions and saving the datastore')
 455   datastore_stub = apiproxy_stub_map.apiproxy.GetStub('datastore_v3')
 456   datastore_stub.Write()
 457
 458
 459 def ParseCommandArguments(args):
 460   """Parses and the application's command line arguments.
 461
 462   Args:
 463     args: A list of command line arguments *not* including the executable or
 464         script e.g. ['-A' 'myapp', '--api_port=8000'].
 465
 466   Returns:
 467     An object containing the values passed in the commandline as attributes.
 468
 469   Raises:
 470     SystemExit: if the argument parsing fails.
 471   """
 472
 473
 474
 475   import argparse
 476   from google.appengine.tools import boolean_action
 477
 478   parser = argparse.ArgumentParser()
 479   parser.add_argument('-A', '--application', required=True)
 480   parser.add_argument('--api_host', default='')
 481
 482   parser.add_argument('--api_port', default=8000, type=int)
 483   parser.add_argument('--trusted',
 484                       action=boolean_action.BooleanAction,
 485                       const=True,
 486                       default=False)
 487   parser.add_argument('--appidentity_email_address', default=None)
 488   parser.add_argument('--appidentity_private_key_path', default=None)
 489   parser.add_argument('--application_root', default=None)
 490   parser.add_argument('--application_host', default='localhost')
 491   parser.add_argument('--application_port', default=None)
 492
 493
 494   parser.add_argument('--blobstore_path', default=None)
 495
 496
 497   parser.add_argument('--datastore_path', default=None)
 498
 499   parser.add_argument('--auto_id_policy', default='scattered',
 500       type=lambda s: s.lower(),
 501       choices=(datastore_stub_util.SEQUENTIAL,
 502                datastore_stub_util.SCATTERED))
 503
 504   parser.add_argument('--use_sqlite',
 505                       action=boolean_action.BooleanAction,
 506                       const=True,
 507                       default=False)
 508   parser.add_argument('--high_replication',
 509                       action=boolean_action.BooleanAction,
 510                       const=True,
 511                       default=False)
 512   parser.add_argument('--require_indexes',
 513                       action=boolean_action.BooleanAction,
 514                       const=True,
 515                       default=False)
 516   parser.add_argument('--clear_datastore',
 517                       action=boolean_action.BooleanAction,
 518                       const=True,
 519                       default=False)
 520
 521
 522   parser.add_argument('--logs_path', default=None)
 523
 524
 525   parser.add_argument('--enable_sendmail',
 526                       action=boolean_action.BooleanAction,
 527                       const=True,
 528                       default=False)
 529   parser.add_argument('--smtp_host', default='')
 530
 531   parser.add_argument('--smtp_port', default=25, type=int)
 532   parser.add_argument('--smtp_user', default='')
 533   parser.add_argument('--smtp_password', default='')
 534   parser.add_argument('--show_mail_body',
 535                       action=boolean_action.BooleanAction,
 536                       const=True,
 537                       default=False)
 538   parser.add_argument('--smtp_allow_tls',
 539                       action=boolean_action.BooleanAction,
 540                       const=True,
 541                       default=True)
 542
 543
 544   parser.add_argument('--prospective_search_path', default=None)
 545   parser.add_argument('--clear_prospective_search',
 546                       action=boolean_action.BooleanAction,
 547                       const=True,
 548                       default=False)
 549
 550
 551   parser.add_argument('--enable_task_running',
 552                       action=boolean_action.BooleanAction,
 553                       const=True,
 554                       default=True)
 555
 556   parser.add_argument('--task_retry_seconds', default=30, type=int)
 557
 558
 559   parser.add_argument('--user_login_url', default=None)
 560   parser.add_argument('--user_logout_url', default=None)
 561
 562   return parser.parse_args(args)
 563
 564
 565 class APIServerProcess(object):
 566   """Manages an API Server running as a seperate process."""
 567
 568
 569
 570
 571
 572
 573   def __init__(self,
 574                executable,
 575                host,
 576                port,
 577                app_id,
 578                script=None,
 579                appidentity_email_address=None,
 580                appidentity_private_key_path=None,
 581                application_host=None,
 582                application_port=None,
 583                application_root=None,
 584                auto_id_policy=None,
 585                blobstore_path=None,
 586                clear_datastore=None,
 587                clear_prospective_search=None,
 588                datastore_path=None,
 589                enable_sendmail=None,
 590                enable_task_running=None,
 591                high_replication=None,
 592                logs_path=None,
 593                prospective_search_path=None,
 594                require_indexes=None,
 595                show_mail_body=None,
 596                smtp_host=None,
 597                smtp_password=None,
 598                smtp_port=None,
 599                smtp_user=None,
 600                smtp_allow_tls=None,
 601                task_retry_seconds=None,
 602                trusted=None,
 603                use_sqlite=None,
 604                default_gcs_bucket_name=None):
 605     """Configures the APIs hosted by this server.
 606
 607     Args:
 608       executable: The path of the executable to use when running the API Server
 609           e.g. "/usr/bin/python".
 610       host: The host name that should be used by the API Server e.g.
 611           "localhost".
 612       port: The port number that should be used by the API Server e.g. 8080.
 613       app_id: The str application id e.g. "guestbook".
 614       script: The name of the script that should be used, along with the
 615           executable argument, to run the API Server e.g. "api_server.py".
 616           If None then the executable is run without a script argument.
 617       appidentity_email_address: Email address for service account substitute.
 618       appidentity_private_key_path: Private key for service account substitute.
 619       application_host: The name of the host where the development application
 620           server is running e.g. "localhost".
 621       application_port: The port where the application server is running e.g.
 622           8000.
 623       application_root: The path to the directory containing the user's
 624           application e.g. "/home/bquinlan/myapp".
 625       auto_id_policy: One of "sequential" or "scattered", indicating whether
 626         the Datastore stub should assign IDs sequentially or scattered.
 627       blobstore_path: The path to the file that should be used for blobstore
 628           storage.
 629       clear_datastore: Clears the file at datastore_path, emptying the
 630           datastore from previous runs.
 631       clear_prospective_search: Clears the file at prospective_search_path,
 632           emptying the perspective search state from previous runs.
 633       datastore_path: The path to the file that should be used for datastore
 634           storage.
 635       enable_sendmail: A bool indicating if sendmail should be used when sending
 636           e-mails. This argument is ignored if mail_smtp_host is not None.
 637       enable_task_running: A bool indicating whether taskqueue tasks should
 638           be run automatically or it the must be manually triggered.
 639       high_replication: A bool indicating whether to use the high replication
 640           consistency model.
 641       logs_path: Path to the file to store the logs data in.
 642       prospective_search_path: The path to the file that should be used to
 643           save prospective search subscriptions.
 644       require_indexes: A bool indicating if the same production
 645           datastore indexes requirements should be enforced i.e. if True then
 646           a google.appengine.ext.db.NeedIndexError will be be raised if a query
 647           is executed without the required indexes.
 648       show_mail_body: A bool indicating whether the body of sent e-mails
 649         should be written to the logs.
 650       smtp_host: The SMTP hostname that should be used when sending e-mails.
 651           If None then the enable_sendmail argument is considered.
 652       smtp_password: The password to use when authenticating with the
 653           SMTP server. This value may be None if smtp_host or smtp_user
 654           is also None.
 655       smtp_port: The SMTP port number that should be used when sending
 656           e-mails. If this value is None then smtp_host must also be None.
 657       smtp_user: The username to use when authenticating with the
 658           SMTP server. This value may be None if smtp_host is also None or if
 659           the SMTP server does not require authentication.
 660       smtp_allow_tls: A bool indicating whether to enable TLS.
 661       task_retry_seconds: An int representing the number of seconds to
 662           wait before a retrying a failed taskqueue task.
 663       trusted: A bool indicating if privileged APIs should be made available.
 664       use_sqlite: A bool indicating whether DatastoreSqliteStub or
 665           DatastoreFileStub should be used.
 666       default_gcs_bucket_name: A str overriding the normal default bucket name.
 667     """
 668     self._process = None
 669     self._host = host
 670     self._port = port
 671     if script:
 672       self._args = [executable, script]
 673     else:
 674       self._args = [executable]
 675     self._BindArgument('--api_host', host)
 676     self._BindArgument('--api_port', port)
 677     self._BindArgument('--appidentity_email_address', appidentity_email_address)
 678     self._BindArgument('--appidentity_private_key_path', appidentity_private_key_path)
 679     self._BindArgument('--application_host', application_host)
 680     self._BindArgument('--application_port', application_port)
 681     self._BindArgument('--application_root', application_root)
 682     self._BindArgument('--application', app_id)
 683     self._BindArgument('--auto_id_policy', auto_id_policy)
 684     self._BindArgument('--blobstore_path', blobstore_path)
 685     self._BindArgument('--clear_datastore', clear_datastore)
 686     self._BindArgument('--clear_prospective_search', clear_prospective_search)
 687     self._BindArgument('--datastore_path', datastore_path)
 688     self._BindArgument('--enable_sendmail', enable_sendmail)
 689     self._BindArgument('--enable_task_running', enable_task_running)
 690     self._BindArgument('--high_replication', high_replication)
 691     self._BindArgument('--logs_path', logs_path)
 692     self._BindArgument('--prospective_search_path', prospective_search_path)
 693     self._BindArgument('--require_indexes', require_indexes)
 694     self._BindArgument('--show_mail_body', show_mail_body)
 695     self._BindArgument('--smtp_host', smtp_host)
 696     self._BindArgument('--smtp_password', smtp_password)
 697     self._BindArgument('--smtp_port', smtp_port)
 698     self._BindArgument('--smtp_user', smtp_user)
 699     self._BindArgument('--smtp_allow_tls', smtp_allow_tls)
 700     self._BindArgument('--task_retry_seconds', task_retry_seconds)
 701     self._BindArgument('--trusted', trusted)
 702     self._BindArgument('--use_sqlite', use_sqlite)
 703     self._BindArgument('--default_gcs_bucket_name', default_gcs_bucket_name)
 704
 705   @property
 706   def url(self):
 707     """Returns the URL that should be used to communicate with the server."""
 708     return 'http://%s:%d' % (self._host, self._port)
 709
 710   def __repr__(self):
 711     return '<APIServerProcess command=%r>' % ' '.join(self._args)
 712
 713   def Start(self):
 714     """Starts the API Server process."""
 715
 716
 717
 718     assert not self._process, 'Start() can only be called once'
 719     self._process = subprocess.Popen(self._args)
 720
 721   def _CanConnect(self):
 722     s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
 723     try:
 724       s.connect((self._host, self._port))
 725     except socket.error:
 726       connected = False
 727     else:
 728       connected = True
 729     s.close()
 730     return connected
 731
 732   def WaitUntilServing(self, timeout=30.0):
 733     """Waits until the API Server is ready to handle requests.
 734
 735     Args:
 736       timeout: The maximum number of seconds to wait for the server to be ready.
 737
 738     Raises:
 739       Error: if the server process exits or is not ready in "timeout" seconds.
 740     """
 741     assert self._process, 'server was not started'
 742     finish_time = time.time() + timeout
 743     while time.time() < finish_time:
 744       if self._process.poll() is not None:
 745         raise Error('server has already exited with return: %r',
 746                     self._process.returncode)
 747       if self._CanConnect():
 748         return
 749       time.sleep(0.2)
 750     raise Error('server did not start after %f seconds', timeout)
 751
 752   def _BindArgument(self, argument, value):
 753     if value is not None:
 754       self._args.append('%s=%s' % (argument, value))
 755
 756   def Quit(self, timeout=5.0):
 757     """Causes the API Server process to exit.
 758
 759     Args:
 760       timeout: The maximum number of seconds to wait for an orderly shutdown
 761           before forceably killing the process.
 762     """
 763     assert self._process, 'server was not started'
 764     if self._process.poll() is None:
 765       try:
 766         urllib2.urlopen(self.url + QUIT_PATH)
 767       except urllib2.URLError:
 768
 769
 770         pass
 771
 772       finish_time = time.time() + timeout
 773       while time.time() < finish_time and self._process.poll() is None:
 774         time.sleep(0.2)
 775       if self._process.returncode is None:
 776         logging.warning('api_server did not quit cleanly, killing')
 777         self._process.kill()
 778
 779
 780 class ApiServerDispatcher(request_info._LocalFakeDispatcher):
 781   """An api_server Dispatcher implementation."""
 782
 783   def add_request(self, method, relative_url, headers, body, source_ip,
 784                   server_name=None, version=None, instance_id=None):
 785     """Process an HTTP request.
 786
 787     Args:
 788       method: A str containing the HTTP method of the request.
 789       relative_url: A str containing path and query string of the request.
 790       headers: A list of (key, value) tuples where key and value are both str.
 791       body: A str containing the request body.
 792       source_ip: The source ip address for the request.
 793       server_name: An optional str containing the server name to service this
 794           request. If unset, the request will be dispatched to the default
 795           server.
 796       version: An optional str containing the version to service this request.
 797           If unset, the request will be dispatched to the default version.
 798       instance_id: An optional str containing the instance_id of the instance to
 799           service this request. If unset, the request will be dispatched to
 800           according to the load-balancing for the server and version.
 801
 802     Returns:
 803       A request_info.ResponseTuple containing the response information for the
 804       HTTP request.
 805     """
 806     try:
 807       header_dict = wsgiref.headers.Headers(headers)
 808       connection_host = header_dict.get('host')
 809       connection = httplib.HTTPConnection(connection_host)
 810
 811
 812       connection.putrequest(
 813           method, relative_url,
 814           skip_host='host' in header_dict,
 815           skip_accept_encoding='accept-encoding' in header_dict)
 816
 817       for header_key, header_value in headers:
 818         connection.putheader(header_key, header_value)
 819       connection.endheaders()
 820       connection.send(body)
 821
 822       response = connection.getresponse()
 823       response.read()
 824       response.close()
 825
 826       return request_info.ResponseTuple(
 827           '%d %s' % (response.status, response.reason), [], '')
 828     except (httplib.HTTPException, socket.error):
 829       logging.exception(
 830           'An error occured while sending a %s request to "%s%s"',
 831           method, connection_host, relative_url)
 832       return request_info.ResponseTuple('0', [], '')
 833
 834
 835 def main():
 836
 837   logging.basicConfig(
 838       level=logging.INFO,
 839       format='[API Server] [%(filename)s:%(lineno)d] %(levelname)s %(message)s')
 840
 841   args = ParseCommandArguments(sys.argv[1:])
 842
 843   if args.clear_datastore:
 844     _ClearDatastoreStorage(args.datastore_path)
 845
 846   if args.clear_prospective_search:
 847     _ClearProspectiveSearchStorage(args.prospective_search_path)
 848
 849   if args.blobstore_path is None:
 850     _, blobstore_temp_filename = tempfile.mkstemp(prefix='ae-blobstore')
 851     args.blobstore_path = blobstore_temp_filename
 852
 853   if args.datastore_path is None:
 854     _, datastore_temp_filename = tempfile.mkstemp(prefix='ae-datastore')
 855     args.datastore_path = datastore_temp_filename
 856
 857   if args.prospective_search_path is None:
 858     _, prospective_search_temp_filename = tempfile.mkstemp(
 859         prefix='ae-prospective_search')
 860     args.prospective_search_path = prospective_search_temp_filename
 861
 862   if args.application_host:
 863     application_address = args.application_host
 864     if args.application_port and args.application_port != 80:
 865       application_address += ':' + str(args.application_port)
 866   else:
 867     application_address = None
 868
 869   if not hasattr(args, 'default_gcs_bucket_name'):
 870     args.default_gcs_bucket_name = None
 871
 872   request_info._local_dispatcher = ApiServerDispatcher()
 873   _SetupStubs(app_id=args.application,
 874               application_root=args.application_root,
 875               appidentity_email_address=args.appidentity_email_address,
 876               appidentity_private_key_path=args.appidentity_private_key_path,
 877               trusted=args.trusted,
 878               blobstore_path=args.blobstore_path,
 879               datastore_path=args.datastore_path,
 880               use_sqlite=args.use_sqlite,
 881               auto_id_policy=args.auto_id_policy,
 882               high_replication=args.high_replication,
 883               datastore_require_indexes=args.require_indexes,
 884               images_host_prefix=application_address,
 885               logs_path=args.logs_path,
 886               mail_smtp_host=args.smtp_host,
 887               mail_smtp_port=args.smtp_port,
 888               mail_smtp_user=args.smtp_user,
 889               mail_smtp_password=args.smtp_password,
 890               mail_enable_sendmail=args.enable_sendmail,
 891               mail_show_mail_body=args.show_mail_body,
 892               mail_allow_tls=args.smtp_allow_tls,
 893               matcher_prospective_search_path=args.prospective_search_path,
 894               taskqueue_auto_run_tasks=args.enable_task_running,
 895               taskqueue_task_retry_seconds=args.task_retry_seconds,
 896               taskqueue_default_http_server=application_address,
 897               user_login_url=args.user_login_url,
 898               user_logout_url=args.user_logout_url,
 899               default_gcs_bucket_name=args.default_gcs_bucket_name)
 900   server = APIServer((args.api_host, args.api_port), args.application)
 901   try:
 902     server.serve_forever()
 903   finally:
 904     _TearDownStubs()
 905
 906
 907 if __name__ == '__main__':
 908   try:
 909     main()
 910   except KeyboardInterrupt:
 911     pass