python/google/appengine/tools/appengine_rpc.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2007 Google Inc.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #
  17 """Tool for performing authenticated RPCs against App Engine."""
  18
  19
  20
  21 import google
  22
  23 import cookielib
  24 import cStringIO
  25 import fancy_urllib
  26 import gzip
  27 import logging
  28 import os
  29 import re
  30 import socket
  31 import sys
  32 import time
  33 import urllib
  34 import urllib2
  35
  36 from google.appengine.tools import dev_appserver_login
  37
  38 _UPLOADING_APP_DOC_URLS = {
  39     "go": "https://developers.google.com/appengine/docs/go/tools/"
  40         "uploadinganapp#Go_Password-less_login_with_OAuth2",
  41     "php": "https://developers.google.com/appengine/docs/php/tools/"
  42         "uploadinganapp#PHP_Password-less_login_with_OAuth2",
  43     "python": "https://developers.google.com/appengine/docs/python/tools/"
  44         "uploadinganapp#Python_Password-less_login_with_OAuth2",
  45     "python27": "https://developers.google.com/appengine/docs/python/tools/"
  46         "uploadinganapp#Python_Password-less_login_with_OAuth2",
  47     "java": "https://developers.google.com/appengine/docs/java/tools/"
  48         "uploadinganapp#Passwordless_Login_with_OAuth2",
  49     "java7": "https://developers.google.com/appengine/docs/java/tools/"
  50         "uploadinganapp#Passwordless_Login_with_OAuth2",
  51     }
  52
  53 logger = logging.getLogger('google.appengine.tools.appengine_rpc')
  54
  55 def GetPlatformToken(os_module=os, sys_module=sys, platform=sys.platform):
  56   """Returns a 'User-agent' token for the host system platform.
  57
  58   Args:
  59     os_module, sys_module, platform: Used for testing.
  60
  61   Returns:
  62     String containing the platform token for the host system.
  63   """
  64   if hasattr(sys_module, "getwindowsversion"):
  65     windows_version = sys_module.getwindowsversion()
  66     version_info = ".".join(str(i) for i in windows_version[:4])
  67     return platform + "/" + version_info
  68   elif hasattr(os_module, "uname"):
  69     uname = os_module.uname()
  70     return "%s/%s" % (uname[0], uname[2])
  71   else:
  72     return "unknown"
  73
  74 def HttpRequestToString(req, include_data=True):
  75   """Converts a urllib2.Request to a string.
  76
  77   Args:
  78     req: urllib2.Request
  79   Returns:
  80     Multi-line string representing the request.
  81   """
  82
  83   headers = ""
  84   for header in req.header_items():
  85     headers += "%s: %s\n" % (header[0], header[1])
  86
  87   template = ("%(method)s %(selector)s %(type)s/1.1\n"
  88               "Host: %(host)s\n"
  89               "%(headers)s")
  90   if include_data:
  91     template = template + "\n%(data)s"
  92
  93   return template % {
  94       'method': req.get_method(),
  95       'selector': req.get_selector(),
  96       'type': req.get_type().upper(),
  97       'host': req.get_host(),
  98       'headers': headers,
  99       'data': req.get_data(),
 100       }
 101
 102 class ClientLoginError(urllib2.HTTPError):
 103   """Raised to indicate there was an error authenticating with ClientLogin."""
 104
 105   def __init__(self, url, code, msg, headers, args):
 106     urllib2.HTTPError.__init__(self, url, code, msg, headers, None)
 107     self.args = args
 108     self._reason = args.get("Error")
 109     self.info = args.get("Info")
 110
 111   def read(self):
 112     return '%d %s: %s' % (self.code, self.msg, self.reason)
 113
 114
 115
 116   @property
 117   def reason(self):
 118     return self._reason
 119
 120
 121 class AbstractRpcServer(object):
 122   """Provides a common interface for a simple RPC server."""
 123
 124
 125   SUGGEST_OAUTH2 = False
 126
 127
 128   RUNTIME = "python"
 129
 130   def __init__(self, host, auth_function, user_agent, source,
 131                host_override=None, extra_headers=None, save_cookies=False,
 132                auth_tries=3, account_type=None, debug_data=True, secure=True,
 133                ignore_certs=False, rpc_tries=3):
 134     """Creates a new HttpRpcServer.
 135
 136     Args:
 137       host: The host to send requests to.
 138       auth_function: A function that takes no arguments and returns an
 139         (email, password) tuple when called. Will be called if authentication
 140         is required.
 141       user_agent: The user-agent string to send to the server. Specify None to
 142         omit the user-agent header.
 143       source: The source to specify in authentication requests.
 144       host_override: The host header to send to the server (defaults to host).
 145       extra_headers: A dict of extra headers to append to every request. Values
 146         supplied here will override other default headers that are supplied.
 147       save_cookies: If True, save the authentication cookies to local disk.
 148         If False, use an in-memory cookiejar instead.  Subclasses must
 149         implement this functionality.  Defaults to False.
 150       auth_tries: The number of times to attempt auth_function before failing.
 151       account_type: One of GOOGLE, HOSTED_OR_GOOGLE, or None for automatic.
 152       debug_data: Whether debugging output should include data contents.
 153       secure: If the requests sent using Send should be sent over HTTPS.
 154       ignore_certs: If the certificate mismatches should be ignored.
 155       rpc_tries: The number of rpc retries upon http server error (i.e.
 156         Response code >= 500 and < 600) before failing.
 157     """
 158     if secure:
 159       self.scheme = "https"
 160     else:
 161       self.scheme = "http"
 162     self.ignore_certs = ignore_certs
 163     self.host = host
 164     self.host_override = host_override
 165     self.auth_function = auth_function
 166     self.source = source
 167     self.authenticated = False
 168     self.auth_tries = auth_tries
 169     self.debug_data = debug_data
 170     self.rpc_tries = rpc_tries
 171
 172
 173     self.account_type = account_type
 174
 175     self.extra_headers = {}
 176     if user_agent:
 177       self.extra_headers["User-Agent"] = user_agent
 178     if extra_headers:
 179       self.extra_headers.update(extra_headers)
 180
 181     self.save_cookies = save_cookies
 182
 183     self.cookie_jar = cookielib.MozillaCookieJar()
 184     self.opener = self._GetOpener()
 185     if self.host_override:
 186       logger.debug("Server: %s; Host: %s", self.host, self.host_override)
 187     else:
 188       logger.debug("Server: %s", self.host)
 189
 190
 191     if ((self.host_override and self.host_override == "localhost") or
 192         self.host == "localhost" or self.host.startswith("localhost:")):
 193       self._DevAppServerAuthenticate()
 194
 195   def _GetOpener(self):
 196     """Returns an OpenerDirector for making HTTP requests.
 197
 198     Returns:
 199       A urllib2.OpenerDirector object.
 200     """
 201     raise NotImplementedError
 202
 203   def _CreateRequest(self, url, data=None):
 204     """Creates a new urllib request."""
 205     req = fancy_urllib.FancyRequest(url, data=data)
 206     if self.host_override:
 207       req.add_header("Host", self.host_override)
 208     for key, value in self.extra_headers.iteritems():
 209       req.add_header(key, value)
 210     return req
 211
 212   def _GetAuthToken(self, email, password):
 213     """Uses ClientLogin to authenticate the user, returning an auth token.
 214
 215     Args:
 216       email:    The user's email address
 217       password: The user's password
 218
 219     Raises:
 220       ClientLoginError: If there was an error authenticating with ClientLogin.
 221       HTTPError: If there was some other form of HTTP error.
 222
 223     Returns:
 224       The authentication token returned by ClientLogin.
 225     """
 226     account_type = self.account_type
 227     if not account_type:
 228
 229       if (self.host.split(':')[0].endswith(".google.com")
 230           or (self.host_override
 231               and self.host_override.split(':')[0].endswith(".google.com"))):
 232
 233         account_type = "HOSTED_OR_GOOGLE"
 234       else:
 235         account_type = "GOOGLE"
 236     data = {
 237         "Email": email,
 238         "Passwd": password,
 239         "service": "ah",
 240         "source": self.source,
 241         "accountType": account_type
 242     }
 243
 244
 245     req = self._CreateRequest(
 246         url=("https://%s/accounts/ClientLogin" %
 247              os.getenv("APPENGINE_AUTH_SERVER", "www.google.com")),
 248         data=urllib.urlencode(data))
 249     try:
 250       response = self.opener.open(req)
 251       response_body = response.read()
 252       response_dict = dict(x.split("=")
 253                            for x in response_body.split("\n") if x)
 254       if os.getenv("APPENGINE_RPC_USE_SID", "0") == "1":
 255         self.extra_headers["Cookie"] = (
 256             'SID=%s; Path=/;' % response_dict["SID"])
 257       return response_dict["Auth"]
 258     except urllib2.HTTPError, e:
 259       if e.code == 403:
 260         body = e.read()
 261         response_dict = dict(x.split("=", 1) for x in body.split("\n") if x)
 262         raise ClientLoginError(req.get_full_url(), e.code, e.msg,
 263                                e.headers, response_dict)
 264       else:
 265         raise
 266
 267   def _GetAuthCookie(self, auth_token):
 268     """Fetches authentication cookies for an authentication token.
 269
 270     Args:
 271       auth_token: The authentication token returned by ClientLogin.
 272
 273     Raises:
 274       HTTPError: If there was an error fetching the authentication cookies.
 275     """
 276
 277     continue_location = "http://localhost/"
 278     args = {"continue": continue_location, "auth": auth_token}
 279     login_path = os.environ.get("APPCFG_LOGIN_PATH", "/_ah")
 280     req = self._CreateRequest("%s://%s%s/login?%s" %
 281                               (self.scheme, self.host, login_path,
 282                                urllib.urlencode(args)))
 283     try:
 284       response = self.opener.open(req)
 285     except urllib2.HTTPError, e:
 286       response = e
 287     if (response.code != 302 or
 288         response.info()["location"] != continue_location):
 289       raise urllib2.HTTPError(req.get_full_url(), response.code, response.msg,
 290                               response.headers, response.fp)
 291     self.authenticated = True
 292
 293   def _Authenticate(self):
 294     """Authenticates the user.
 295
 296     The authentication process works as follows:
 297      1) We get a username and password from the user
 298      2) We use ClientLogin to obtain an AUTH token for the user
 299         (see http://code.google.com/apis/accounts/AuthForInstalledApps.html).
 300      3) We pass the auth token to /_ah/login on the server to obtain an
 301         authentication cookie. If login was successful, it tries to redirect
 302         us to the URL we provided.
 303
 304     If we attempt to access the upload API without first obtaining an
 305     authentication cookie, it returns a 401 response and directs us to
 306     authenticate ourselves with ClientLogin.
 307     """
 308     for unused_i in range(self.auth_tries):
 309       credentials = self.auth_function()
 310       try:
 311         auth_token = self._GetAuthToken(credentials[0], credentials[1])
 312         if os.getenv("APPENGINE_RPC_USE_SID", "0") == "1":
 313           return
 314       except ClientLoginError, e:
 315         if e.reason == "BadAuthentication":
 316           if e.info == "InvalidSecondFactor":
 317             print >>sys.stderr, ("Use an application-specific password instead "
 318                                  "of your regular account password.")
 319             print >>sys.stderr, ("See http://www.google.com/"
 320                                  "support/accounts/bin/answer.py?answer=185833")
 321
 322
 323
 324             if self.SUGGEST_OAUTH2:
 325               print >>sys.stderr, ("However, now the recommended way to log in "
 326                                    "is using OAuth2. See")
 327               print >>sys.stderr, _UPLOADING_APP_DOC_URLS[self.RUNTIME]
 328           else:
 329             print >>sys.stderr, "Invalid username or password."
 330           continue
 331         if e.reason == "CaptchaRequired":
 332           print >>sys.stderr, (
 333               "Please go to\n"
 334               "https://www.google.com/accounts/DisplayUnlockCaptcha\n"
 335               "and verify you are a human.  Then try again.")
 336           break
 337         if e.reason == "NotVerified":
 338           print >>sys.stderr, "Account not verified."
 339           break
 340         if e.reason == "TermsNotAgreed":
 341           print >>sys.stderr, "User has not agreed to TOS."
 342           break
 343         if e.reason == "AccountDeleted":
 344           print >>sys.stderr, "The user account has been deleted."
 345           break
 346         if e.reason == "AccountDisabled":
 347           print >>sys.stderr, "The user account has been disabled."
 348           break
 349         if e.reason == "ServiceDisabled":
 350           print >>sys.stderr, ("The user's access to the service has been "
 351                                "disabled.")
 352           break
 353         if e.reason == "ServiceUnavailable":
 354           print >>sys.stderr, "The service is not available; try again later."
 355           break
 356         raise
 357       self._GetAuthCookie(auth_token)
 358       return
 359
 360   def _DevAppServerAuthenticate(self):
 361     """Authenticates the user on the dev_appserver."""
 362     credentials = self.auth_function()
 363     value = dev_appserver_login.CreateCookieData(credentials[0], True)
 364     self.extra_headers["Cookie"] = ('dev_appserver_login="%s"; Path=/;' % value)
 365
 366   def Send(self, request_path, payload="",
 367            content_type="application/octet-stream",
 368            timeout=None,
 369            **kwargs):
 370     """Sends an RPC and returns the response.
 371
 372     Args:
 373       request_path: The path to send the request to, eg /api/appversion/create.
 374       payload: The body of the request, or None to send an empty request.
 375       content_type: The Content-Type header to use.
 376       timeout: timeout in seconds; default None i.e. no timeout.
 377         (Note: for large requests on OS X, the timeout doesn't work right.)
 378       kwargs: Any keyword arguments are converted into query string parameters.
 379
 380     Returns:
 381       The response body, as a string.
 382     """
 383     old_timeout = socket.getdefaulttimeout()
 384     socket.setdefaulttimeout(timeout)
 385     try:
 386       tries = 0
 387       auth_tried = False
 388       while True:
 389         tries += 1
 390         url = "%s://%s%s" % (self.scheme, self.host, request_path)
 391         if kwargs:
 392
 393
 394           url += "?" + urllib.urlencode(sorted(kwargs.items()))
 395         req = self._CreateRequest(url=url, data=payload)
 396         req.add_header("Content-Type", content_type)
 397
 398
 399
 400         req.add_header("X-appcfg-api-version", "1")
 401
 402         try:
 403           logger.debug('Sending %s request:\n%s',
 404                        self.scheme.upper(),
 405                        HttpRequestToString(req, include_data=self.debug_data))
 406           f = self.opener.open(req)
 407           response = f.read()
 408           f.close()
 409
 410           return response
 411         except urllib2.HTTPError, e:
 412           logger.debug("Got http error, this is try #%s", tries)
 413           if tries > self.rpc_tries:
 414             raise
 415           elif e.code == 401:
 416
 417             if auth_tried:
 418               raise
 419             auth_tried = True
 420             self._Authenticate()
 421           elif e.code >= 500 and e.code < 600:
 422
 423             continue
 424           elif e.code == 302:
 425
 426
 427             if auth_tried:
 428               raise
 429             auth_tried = True
 430             loc = e.info()["location"]
 431             logger.debug("Got 302 redirect. Location: %s", loc)
 432             if loc.startswith("https://www.google.com/accounts/ServiceLogin"):
 433               self._Authenticate()
 434             elif re.match(
 435                 r"https://www\.google\.com/a/[a-z0-9\.\-]+/ServiceLogin", loc):
 436               self.account_type = os.getenv("APPENGINE_RPC_HOSTED_LOGIN_TYPE",
 437                                             "HOSTED")
 438               self._Authenticate()
 439             elif loc.startswith("http://%s/_ah/login" % (self.host,)):
 440               self._DevAppServerAuthenticate()
 441             else:
 442               raise
 443           else:
 444             raise
 445     finally:
 446       socket.setdefaulttimeout(old_timeout)
 447
 448
 449 class ContentEncodingHandler(urllib2.BaseHandler):
 450   """Request and handle HTTP Content-Encoding."""
 451   def http_request(self, request):
 452
 453     request.add_header("Accept-Encoding", "gzip")
 454
 455
 456
 457
 458
 459
 460
 461
 462
 463
 464
 465
 466     for header in request.headers:
 467       if header.lower() == "user-agent":
 468         request.headers[header] += " gzip"
 469
 470     return request
 471
 472   https_request = http_request
 473
 474   def http_response(self, req, resp):
 475     """Handle encodings in the order that they are encountered."""
 476     encodings = []
 477     headers = resp.headers
 478
 479     for header in headers:
 480       if header.lower() == "content-encoding":
 481         for encoding in headers.get(header, "").split(","):
 482           encoding = encoding.strip()
 483           if encoding:
 484             encodings.append(encoding)
 485         break
 486
 487     if not encodings:
 488       return resp
 489
 490     del headers[header]
 491
 492     fp = resp
 493     while encodings and encodings[-1].lower() == "gzip":
 494       fp = cStringIO.StringIO(fp.read())
 495       fp = gzip.GzipFile(fileobj=fp, mode="r")
 496       encodings.pop()
 497
 498     if encodings:
 499
 500
 501
 502
 503       headers[header] = ", ".join(encodings)
 504       logger.warning("Unrecognized Content-Encoding: %s", encodings[-1])
 505
 506     msg = resp.msg
 507     if sys.version_info >= (2, 6):
 508       resp = urllib2.addinfourl(fp, headers, resp.url, resp.code)
 509     else:
 510       response_code = resp.code
 511       resp = urllib2.addinfourl(fp, headers, resp.url)
 512       resp.code = response_code
 513     resp.msg = msg
 514
 515     return resp
 516
 517   https_response = http_response
 518
 519
 520 class HttpRpcServer(AbstractRpcServer):
 521   """Provides a simplified RPC-style interface for HTTP requests."""
 522
 523   DEFAULT_COOKIE_FILE_PATH = "~/.appcfg_cookies"
 524
 525   def __init__(self, *args, **kwargs):
 526     self.certpath = os.path.normpath(os.path.join(
 527         os.path.dirname(__file__), '..', '..', '..', 'lib', 'cacerts',
 528         'cacerts.txt'))
 529     self.cert_file_available = ((not kwargs.get("ignore_certs", False))
 530                                 and os.path.exists(self.certpath))
 531     super(HttpRpcServer, self).__init__(*args, **kwargs)
 532
 533   def _CreateRequest(self, url, data=None):
 534     """Creates a new urllib request."""
 535     req = super(HttpRpcServer, self)._CreateRequest(url, data)
 536     if self.cert_file_available and fancy_urllib.can_validate_certs():
 537       req.set_ssl_info(ca_certs=self.certpath)
 538     return req
 539
 540   def _CheckCookie(self):
 541     """Warn if cookie is not valid for at least one minute."""
 542     min_expire = time.time() + 60
 543
 544     for cookie in self.cookie_jar:
 545       if cookie.domain == self.host and not cookie.is_expired(min_expire):
 546         break
 547     else:
 548       print >>sys.stderr, "\nError: Machine system clock is incorrect.\n"
 549
 550
 551   def _Authenticate(self):
 552     """Save the cookie jar after authentication."""
 553     if self.cert_file_available and not fancy_urllib.can_validate_certs():
 554
 555
 556       logger.warn("""ssl module not found.
 557 Without the ssl module, the identity of the remote host cannot be verified, and
 558 connections may NOT be secure. To fix this, please install the ssl module from
 559 http://pypi.python.org/pypi/ssl .
 560 To learn more, see https://developers.google.com/appengine/kb/general#rpcssl""")
 561     super(HttpRpcServer, self)._Authenticate()
 562     if self.cookie_jar.filename is not None and self.save_cookies:
 563       logger.debug("Saving authentication cookies to %s",
 564                    self.cookie_jar.filename)
 565       self.cookie_jar.save()
 566       self._CheckCookie()
 567
 568   def _GetOpener(self):
 569     """Returns an OpenerDirector that supports cookies and ignores redirects.
 570
 571     Returns:
 572       A urllib2.OpenerDirector object.
 573     """
 574     opener = urllib2.OpenerDirector()
 575     opener.add_handler(fancy_urllib.FancyProxyHandler())
 576     opener.add_handler(urllib2.UnknownHandler())
 577     opener.add_handler(urllib2.HTTPHandler())
 578     opener.add_handler(urllib2.HTTPDefaultErrorHandler())
 579     opener.add_handler(fancy_urllib.FancyHTTPSHandler())
 580     opener.add_handler(urllib2.HTTPErrorProcessor())
 581     opener.add_handler(ContentEncodingHandler())
 582
 583     if self.save_cookies:
 584       self.cookie_jar.filename = os.path.expanduser(
 585           HttpRpcServer.DEFAULT_COOKIE_FILE_PATH)
 586
 587       if os.path.exists(self.cookie_jar.filename):
 588         try:
 589           self.cookie_jar.load()
 590           self.authenticated = True
 591           logger.debug("Loaded authentication cookies from %s",
 592                        self.cookie_jar.filename)
 593         except (OSError, IOError, cookielib.LoadError), e:
 594
 595           logger.debug("Could not load authentication cookies; %s: %s",
 596                        e.__class__.__name__, e)
 597           self.cookie_jar.filename = None
 598       else:
 599
 600
 601         try:
 602           fd = os.open(self.cookie_jar.filename, os.O_CREAT, 0600)
 603           os.close(fd)
 604         except (OSError, IOError), e:
 605
 606           logger.debug("Could not create authentication cookies file; %s: %s",
 607                        e.__class__.__name__, e)
 608           self.cookie_jar.filename = None
 609
 610     opener.add_handler(urllib2.HTTPCookieProcessor(self.cookie_jar))
 611     return opener
 612
 613
 614
 615 class HttpRpcServerWithOAuth2Suggestion(HttpRpcServer):
 616   """An HttpRpcServer variant which suggests using OAuth2 instead of ASP.
 617
 618   Not all systems which use HttpRpcServer can use OAuth2.
 619   """
 620
 621   SUGGEST_OAUTH2 = True