python/google/appengine/tools/appengine_rpc.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2007 Google Inc.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #
  17
  18
  19
  20
  21 """Tool for performing authenticated RPCs against App Engine."""
  22
  23
  24 import google
  25
  26 import cookielib
  27 import cStringIO
  28 import fancy_urllib
  29 import gzip
  30 import logging
  31 import os
  32 import re
  33 import socket
  34 import sys
  35 import time
  36 import urllib
  37 import urllib2
  38
  39 from google.appengine.tools import dev_appserver_login
  40
  41 _UPLOADING_APP_DOC_URLS = {
  42     "go": "https://developers.google.com/appengine/docs/go/tools/"
  43         "uploadinganapp#Go_Password-less_login_with_OAuth2",
  44     "php": "https://developers.google.com/appengine/docs/php/tools/"
  45         "uploadinganapp#PHP_Password-less_login_with_OAuth2",
  46     "python": "https://developers.google.com/appengine/docs/python/tools/"
  47         "uploadinganapp#Python_Password-less_login_with_OAuth2",
  48     "python27": "https://developers.google.com/appengine/docs/python/tools/"
  49         "uploadinganapp#Python_Password-less_login_with_OAuth2",
  50     "java": "https://developers.google.com/appengine/docs/java/tools/"
  51         "uploadinganapp#Passwordless_Login_with_OAuth2",
  52     "java7": "https://developers.google.com/appengine/docs/java/tools/"
  53         "uploadinganapp#Passwordless_Login_with_OAuth2",
  54     }
  55
  56 logger = logging.getLogger('google.appengine.tools.appengine_rpc')
  57
  58 def GetPlatformToken(os_module=os, sys_module=sys, platform=sys.platform):
  59   """Returns a 'User-agent' token for the host system platform.
  60
  61   Args:
  62     os_module, sys_module, platform: Used for testing.
  63
  64   Returns:
  65     String containing the platform token for the host system.
  66   """
  67   if hasattr(sys_module, "getwindowsversion"):
  68     windows_version = sys_module.getwindowsversion()
  69     version_info = ".".join(str(i) for i in windows_version[:4])
  70     return platform + "/" + version_info
  71   elif hasattr(os_module, "uname"):
  72     uname = os_module.uname()
  73     return "%s/%s" % (uname[0], uname[2])
  74   else:
  75     return "unknown"
  76
  77 def HttpRequestToString(req, include_data=True):
  78   """Converts a urllib2.Request to a string.
  79
  80   Args:
  81     req: urllib2.Request
  82   Returns:
  83     Multi-line string representing the request.
  84   """
  85
  86   headers = ""
  87   for header in req.header_items():
  88     headers += "%s: %s\n" % (header[0], header[1])
  89
  90   template = ("%(method)s %(selector)s %(type)s/1.1\n"
  91               "Host: %(host)s\n"
  92               "%(headers)s")
  93   if include_data:
  94     template = template + "\n%(data)s"
  95
  96   return template % {
  97       'method': req.get_method(),
  98       'selector': req.get_selector(),
  99       'type': req.get_type().upper(),
 100       'host': req.get_host(),
 101       'headers': headers,
 102       'data': req.get_data(),
 103       }
 104
 105 class ClientLoginError(urllib2.HTTPError):
 106   """Raised to indicate there was an error authenticating with ClientLogin."""
 107
 108   def __init__(self, url, code, msg, headers, args):
 109     urllib2.HTTPError.__init__(self, url, code, msg, headers, None)
 110     self.args = args
 111     self._reason = args.get("Error")
 112     self.info = args.get("Info")
 113
 114   def read(self):
 115     return '%d %s: %s' % (self.code, self.msg, self.reason)
 116
 117
 118
 119   @property
 120   def reason(self):
 121     return self._reason
 122
 123
 124 class AbstractRpcServer(object):
 125   """Provides a common interface for a simple RPC server."""
 126
 127
 128   SUGGEST_OAUTH2 = False
 129
 130
 131   RUNTIME = "python"
 132
 133   def __init__(self, host, auth_function, user_agent, source,
 134                host_override=None, extra_headers=None, save_cookies=False,
 135                auth_tries=3, account_type=None, debug_data=True, secure=True,
 136                ignore_certs=False, rpc_tries=3):
 137     """Creates a new HttpRpcServer.
 138
 139     Args:
 140       host: The host to send requests to.
 141       auth_function: A function that takes no arguments and returns an
 142         (email, password) tuple when called. Will be called if authentication
 143         is required.
 144       user_agent: The user-agent string to send to the server. Specify None to
 145         omit the user-agent header.
 146       source: The source to specify in authentication requests.
 147       host_override: The host header to send to the server (defaults to host).
 148       extra_headers: A dict of extra headers to append to every request. Values
 149         supplied here will override other default headers that are supplied.
 150       save_cookies: If True, save the authentication cookies to local disk.
 151         If False, use an in-memory cookiejar instead.  Subclasses must
 152         implement this functionality.  Defaults to False.
 153       auth_tries: The number of times to attempt auth_function before failing.
 154       account_type: One of GOOGLE, HOSTED_OR_GOOGLE, or None for automatic.
 155       debug_data: Whether debugging output should include data contents.
 156       secure: If the requests sent using Send should be sent over HTTPS.
 157       ignore_certs: If the certificate mismatches should be ignored.
 158       rpc_tries: The number of rpc retries upon http server error (i.e.
 159         Response code >= 500 and < 600) before failing.
 160     """
 161     if secure:
 162       self.scheme = "https"
 163     else:
 164       self.scheme = "http"
 165     self.ignore_certs = ignore_certs
 166     self.host = host
 167     self.host_override = host_override
 168     self.auth_function = auth_function
 169     self.source = source
 170     self.authenticated = False
 171     self.auth_tries = auth_tries
 172     self.debug_data = debug_data
 173     self.rpc_tries = rpc_tries
 174
 175
 176     self.account_type = account_type
 177
 178     self.extra_headers = {}
 179     if user_agent:
 180       self.extra_headers["User-Agent"] = user_agent
 181     if extra_headers:
 182       self.extra_headers.update(extra_headers)
 183
 184     self.save_cookies = save_cookies
 185
 186     self.cookie_jar = cookielib.MozillaCookieJar()
 187     self.opener = self._GetOpener()
 188     if self.host_override:
 189       logger.debug("Server: %s; Host: %s", self.host, self.host_override)
 190     else:
 191       logger.debug("Server: %s", self.host)
 192
 193
 194     if ((self.host_override and self.host_override == "localhost") or
 195         self.host == "localhost" or self.host.startswith("localhost:")):
 196       self._DevAppServerAuthenticate()
 197
 198   def _GetOpener(self):
 199     """Returns an OpenerDirector for making HTTP requests.
 200
 201     Returns:
 202       A urllib2.OpenerDirector object.
 203     """
 204     raise NotImplementedError
 205
 206   def _CreateRequest(self, url, data=None):
 207     """Creates a new urllib request."""
 208     req = fancy_urllib.FancyRequest(url, data=data)
 209     if self.host_override:
 210       req.add_header("Host", self.host_override)
 211     for key, value in self.extra_headers.iteritems():
 212       req.add_header(key, value)
 213     return req
 214
 215   def _GetAuthToken(self, email, password):
 216     """Uses ClientLogin to authenticate the user, returning an auth token.
 217
 218     Args:
 219       email:    The user's email address
 220       password: The user's password
 221
 222     Raises:
 223       ClientLoginError: If there was an error authenticating with ClientLogin.
 224       HTTPError: If there was some other form of HTTP error.
 225
 226     Returns:
 227       The authentication token returned by ClientLogin.
 228     """
 229     account_type = self.account_type
 230     if not account_type:
 231
 232       if (self.host.split(':')[0].endswith(".google.com")
 233           or (self.host_override
 234               and self.host_override.split(':')[0].endswith(".google.com"))):
 235
 236         account_type = "HOSTED_OR_GOOGLE"
 237       else:
 238         account_type = "GOOGLE"
 239     data = {
 240         "Email": email,
 241         "Passwd": password,
 242         "service": "ah",
 243         "source": self.source,
 244         "accountType": account_type
 245     }
 246
 247
 248     req = self._CreateRequest(
 249         url=("https://%s/accounts/ClientLogin" %
 250              os.getenv("APPENGINE_AUTH_SERVER", "www.google.com")),
 251         data=urllib.urlencode(data))
 252     try:
 253       response = self.opener.open(req)
 254       response_body = response.read()
 255       response_dict = dict(x.split("=")
 256                            for x in response_body.split("\n") if x)
 257       if os.getenv("APPENGINE_RPC_USE_SID", "0") == "1":
 258         self.extra_headers["Cookie"] = (
 259             'SID=%s; Path=/;' % response_dict["SID"])
 260       return response_dict["Auth"]
 261     except urllib2.HTTPError, e:
 262       if e.code == 403:
 263         body = e.read()
 264         response_dict = dict(x.split("=", 1) for x in body.split("\n") if x)
 265         raise ClientLoginError(req.get_full_url(), e.code, e.msg,
 266                                e.headers, response_dict)
 267       else:
 268         raise
 269
 270   def _GetAuthCookie(self, auth_token):
 271     """Fetches authentication cookies for an authentication token.
 272
 273     Args:
 274       auth_token: The authentication token returned by ClientLogin.
 275
 276     Raises:
 277       HTTPError: If there was an error fetching the authentication cookies.
 278     """
 279
 280     continue_location = "http://localhost/"
 281     args = {"continue": continue_location, "auth": auth_token}
 282     login_path = os.environ.get("APPCFG_LOGIN_PATH", "/_ah")
 283     req = self._CreateRequest("%s://%s%s/login?%s" %
 284                               (self.scheme, self.host, login_path,
 285                                urllib.urlencode(args)))
 286     try:
 287       response = self.opener.open(req)
 288     except urllib2.HTTPError, e:
 289       response = e
 290     if (response.code != 302 or
 291         response.info()["location"] != continue_location):
 292       raise urllib2.HTTPError(req.get_full_url(), response.code, response.msg,
 293                               response.headers, response.fp)
 294     self.authenticated = True
 295
 296   def _Authenticate(self):
 297     """Authenticates the user.
 298
 299     The authentication process works as follows:
 300      1) We get a username and password from the user
 301      2) We use ClientLogin to obtain an AUTH token for the user
 302         (see http://code.google.com/apis/accounts/AuthForInstalledApps.html).
 303      3) We pass the auth token to /_ah/login on the server to obtain an
 304         authentication cookie. If login was successful, it tries to redirect
 305         us to the URL we provided.
 306
 307     If we attempt to access the upload API without first obtaining an
 308     authentication cookie, it returns a 401 response and directs us to
 309     authenticate ourselves with ClientLogin.
 310     """
 311     for unused_i in range(self.auth_tries):
 312       credentials = self.auth_function()
 313       try:
 314         auth_token = self._GetAuthToken(credentials[0], credentials[1])
 315         if os.getenv("APPENGINE_RPC_USE_SID", "0") == "1":
 316           return
 317       except ClientLoginError, e:
 318         if e.reason == "BadAuthentication":
 319           if e.info == "InvalidSecondFactor":
 320             print >>sys.stderr, ("Use an application-specific password instead "
 321                                  "of your regular account password.")
 322             print >>sys.stderr, ("See http://www.google.com/"
 323                                  "support/accounts/bin/answer.py?answer=185833")
 324
 325
 326
 327             if self.SUGGEST_OAUTH2:
 328               print >>sys.stderr, ("However, now the recommended way to log in "
 329                                    "is using OAuth2. See")
 330               print >>sys.stderr, _UPLOADING_APP_DOC_URLS[self.RUNTIME]
 331           else:
 332             print >>sys.stderr, "Invalid username or password."
 333           continue
 334         if e.reason == "CaptchaRequired":
 335           print >>sys.stderr, (
 336               "Please go to\n"
 337               "https://www.google.com/accounts/DisplayUnlockCaptcha\n"
 338               "and verify you are a human.  Then try again.")
 339           break
 340         if e.reason == "NotVerified":
 341           print >>sys.stderr, "Account not verified."
 342           break
 343         if e.reason == "TermsNotAgreed":
 344           print >>sys.stderr, "User has not agreed to TOS."
 345           break
 346         if e.reason == "AccountDeleted":
 347           print >>sys.stderr, "The user account has been deleted."
 348           break
 349         if e.reason == "AccountDisabled":
 350           print >>sys.stderr, "The user account has been disabled."
 351           break
 352         if e.reason == "ServiceDisabled":
 353           print >>sys.stderr, ("The user's access to the service has been "
 354                                "disabled.")
 355           break
 356         if e.reason == "ServiceUnavailable":
 357           print >>sys.stderr, "The service is not available; try again later."
 358           break
 359         raise
 360       self._GetAuthCookie(auth_token)
 361       return
 362
 363   def _DevAppServerAuthenticate(self):
 364     """Authenticates the user on the dev_appserver."""
 365     credentials = self.auth_function()
 366     value = dev_appserver_login.CreateCookieData(credentials[0], True)
 367     self.extra_headers["Cookie"] = ('dev_appserver_login="%s"; Path=/;' % value)
 368
 369   def Send(self, request_path, payload="",
 370            content_type="application/octet-stream",
 371            timeout=None,
 372            **kwargs):
 373     """Sends an RPC and returns the response.
 374
 375     Args:
 376       request_path: The path to send the request to, eg /api/appversion/create.
 377       payload: The body of the request, or None to send an empty request.
 378       content_type: The Content-Type header to use.
 379       timeout: timeout in seconds; default None i.e. no timeout.
 380         (Note: for large requests on OS X, the timeout doesn't work right.)
 381       kwargs: Any keyword arguments are converted into query string parameters.
 382
 383     Returns:
 384       The response body, as a string.
 385     """
 386     old_timeout = socket.getdefaulttimeout()
 387     socket.setdefaulttimeout(timeout)
 388     try:
 389       tries = 0
 390       auth_tried = False
 391       while True:
 392         tries += 1
 393         url = "%s://%s%s" % (self.scheme, self.host, request_path)
 394         if kwargs:
 395
 396
 397           url += "?" + urllib.urlencode(sorted(kwargs.items()))
 398         req = self._CreateRequest(url=url, data=payload)
 399         req.add_header("Content-Type", content_type)
 400
 401
 402
 403         req.add_header("X-appcfg-api-version", "1")
 404
 405         try:
 406           logger.debug('Sending %s request:\n%s',
 407                        self.scheme.upper(),
 408                        HttpRequestToString(req, include_data=self.debug_data))
 409           f = self.opener.open(req)
 410           response = f.read()
 411           f.close()
 412
 413           return response
 414         except urllib2.HTTPError, e:
 415           logger.debug("Got http error, this is try #%s", tries)
 416           if tries > self.rpc_tries:
 417             raise
 418           elif e.code == 401:
 419
 420             if auth_tried:
 421               raise
 422             auth_tried = True
 423             self._Authenticate()
 424           elif e.code >= 500 and e.code < 600:
 425
 426             continue
 427           elif e.code == 302:
 428
 429
 430             if auth_tried:
 431               raise
 432             auth_tried = True
 433             loc = e.info()["location"]
 434             logger.debug("Got 302 redirect. Location: %s", loc)
 435             if loc.startswith("https://www.google.com/accounts/ServiceLogin"):
 436               self._Authenticate()
 437             elif re.match(r"https://www.google.com/a/[a-z0-9.-]+/ServiceLogin",
 438                           loc):
 439               self.account_type = os.getenv("APPENGINE_RPC_HOSTED_LOGIN_TYPE",
 440                                             "HOSTED")
 441               self._Authenticate()
 442             elif loc.startswith("http://%s/_ah/login" % (self.host,)):
 443               self._DevAppServerAuthenticate()
 444             else:
 445               raise
 446           else:
 447             raise
 448     finally:
 449       socket.setdefaulttimeout(old_timeout)
 450
 451
 452 class ContentEncodingHandler(urllib2.BaseHandler):
 453   """Request and handle HTTP Content-Encoding."""
 454   def http_request(self, request):
 455
 456     request.add_header("Accept-Encoding", "gzip")
 457
 458
 459
 460
 461
 462
 463
 464
 465
 466
 467
 468
 469     for header in request.headers:
 470       if header.lower() == "user-agent":
 471         request.headers[header] += " gzip"
 472
 473     return request
 474
 475   https_request = http_request
 476
 477   def http_response(self, req, resp):
 478     """Handle encodings in the order that they are encountered."""
 479     encodings = []
 480     headers = resp.headers
 481
 482     for header in headers:
 483       if header.lower() == "content-encoding":
 484         for encoding in headers.get(header, "").split(","):
 485           encoding = encoding.strip()
 486           if encoding:
 487             encodings.append(encoding)
 488         break
 489
 490     if not encodings:
 491       return resp
 492
 493     del headers[header]
 494
 495     fp = resp
 496     while encodings and encodings[-1].lower() == "gzip":
 497       fp = cStringIO.StringIO(fp.read())
 498       fp = gzip.GzipFile(fileobj=fp, mode="r")
 499       encodings.pop()
 500
 501     if encodings:
 502
 503
 504
 505
 506       headers[header] = ", ".join(encodings)
 507       logger.warning("Unrecognized Content-Encoding: %s", encodings[-1])
 508
 509     msg = resp.msg
 510     if sys.version_info >= (2, 6):
 511       resp = urllib2.addinfourl(fp, headers, resp.url, resp.code)
 512     else:
 513       response_code = resp.code
 514       resp = urllib2.addinfourl(fp, headers, resp.url)
 515       resp.code = response_code
 516     resp.msg = msg
 517
 518     return resp
 519
 520   https_response = http_response
 521
 522
 523 class HttpRpcServer(AbstractRpcServer):
 524   """Provides a simplified RPC-style interface for HTTP requests."""
 525
 526   DEFAULT_COOKIE_FILE_PATH = "~/.appcfg_cookies"
 527
 528   def __init__(self, *args, **kwargs):
 529     self.certpath = os.path.normpath(os.path.join(
 530         os.path.dirname(__file__), '..', '..', '..', 'lib', 'cacerts',
 531         'cacerts.txt'))
 532     self.cert_file_available = ((not kwargs.get("ignore_certs", False))
 533                                 and os.path.exists(self.certpath))
 534     super(HttpRpcServer, self).__init__(*args, **kwargs)
 535
 536   def _CreateRequest(self, url, data=None):
 537     """Creates a new urllib request."""
 538     req = super(HttpRpcServer, self)._CreateRequest(url, data)
 539     if self.cert_file_available and fancy_urllib.can_validate_certs():
 540       req.set_ssl_info(ca_certs=self.certpath)
 541     return req
 542
 543   def _CheckCookie(self):
 544     """Warn if cookie is not valid for at least one minute."""
 545     min_expire = time.time() + 60
 546
 547     for cookie in self.cookie_jar:
 548       if cookie.domain == self.host and not cookie.is_expired(min_expire):
 549         break
 550     else:
 551       print >>sys.stderr, "\nError: Machine system clock is incorrect.\n"
 552
 553
 554   def _Authenticate(self):
 555     """Save the cookie jar after authentication."""
 556     if self.cert_file_available and not fancy_urllib.can_validate_certs():
 557
 558
 559       logger.warn("""ssl module not found.
 560 Without the ssl module, the identity of the remote host cannot be verified, and
 561 connections may NOT be secure. To fix this, please install the ssl module from
 562 http://pypi.python.org/pypi/ssl .
 563 To learn more, see https://developers.google.com/appengine/kb/general#rpcssl""")
 564     super(HttpRpcServer, self)._Authenticate()
 565     if self.cookie_jar.filename is not None and self.save_cookies:
 566       logger.debug("Saving authentication cookies to %s",
 567                    self.cookie_jar.filename)
 568       self.cookie_jar.save()
 569       self._CheckCookie()
 570
 571   def _GetOpener(self):
 572     """Returns an OpenerDirector that supports cookies and ignores redirects.
 573
 574     Returns:
 575       A urllib2.OpenerDirector object.
 576     """
 577     opener = urllib2.OpenerDirector()
 578     opener.add_handler(fancy_urllib.FancyProxyHandler())
 579     opener.add_handler(urllib2.UnknownHandler())
 580     opener.add_handler(urllib2.HTTPHandler())
 581     opener.add_handler(urllib2.HTTPDefaultErrorHandler())
 582     opener.add_handler(fancy_urllib.FancyHTTPSHandler())
 583     opener.add_handler(urllib2.HTTPErrorProcessor())
 584     opener.add_handler(ContentEncodingHandler())
 585
 586     if self.save_cookies:
 587       self.cookie_jar.filename = os.path.expanduser(
 588           HttpRpcServer.DEFAULT_COOKIE_FILE_PATH)
 589
 590       if os.path.exists(self.cookie_jar.filename):
 591         try:
 592           self.cookie_jar.load()
 593           self.authenticated = True
 594           logger.debug("Loaded authentication cookies from %s",
 595                        self.cookie_jar.filename)
 596         except (OSError, IOError, cookielib.LoadError), e:
 597
 598           logger.debug("Could not load authentication cookies; %s: %s",
 599                        e.__class__.__name__, e)
 600           self.cookie_jar.filename = None
 601       else:
 602
 603
 604         try:
 605           fd = os.open(self.cookie_jar.filename, os.O_CREAT, 0600)
 606           os.close(fd)
 607         except (OSError, IOError), e:
 608
 609           logger.debug("Could not create authentication cookies file; %s: %s",
 610                        e.__class__.__name__, e)
 611           self.cookie_jar.filename = None
 612
 613     opener.add_handler(urllib2.HTTPCookieProcessor(self.cookie_jar))
 614     return opener
 615
 616
 617
 618 class HttpRpcServerWithOAuth2Suggestion(HttpRpcServer):
 619   """An HttpRpcServer variant which suggests using OAuth2 instead of ASP.
 620
 621   Not all systems which use HttpRpcServer can use OAuth2.
 622   """
 623
 624   SUGGEST_OAUTH2 = True