python/google/appengine/tools/dev_appserver_blobstore.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2007 Google Inc.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #
  17 """Blobstore support classes.
  18
  19 Classes:
  20
  21   DownloadRewriter:
  22     Rewriter responsible for transforming an application response to one
  23     that serves a blob to the user.
  24
  25   CreateUploadDispatcher:
  26     Creates a dispatcher that is added to dispatcher chain.  Handles uploads
  27     by storing blobs rewriting requests and returning a redirect.
  28 """
  29
  30
  31
  32 import cgi
  33 import cStringIO
  34 import logging
  35 import mimetools
  36 import re
  37
  38 from google.appengine.api import apiproxy_stub_map
  39 from google.appengine.api import blobstore
  40 from google.appengine.api import datastore
  41 from google.appengine.api import datastore_errors
  42 from google.appengine.api.files import file_service_stub
  43 from google.appengine.tools import dev_appserver_upload
  44
  45
  46
  47 UPLOAD_URL_PATH = '_ah/upload/'
  48
  49
  50 UPLOAD_URL_PATTERN = '/%s(.*)' % UPLOAD_URL_PATH
  51
  52
  53 AUTO_MIME_TYPE = 'application/vnd.google.appengine.auto'
  54
  55
  56 ERROR_RESPONSE_TEMPLATE = """
  57 <html>
  58   <head>
  59     <title>%(response_code)d %(response_string)s</title>
  60   </head>
  61   <body text=#000000 bgcolor=#ffffff>
  62     <h1>Error: %(response_string)s</h1>
  63     <h2>%(response_text)s</h2>
  64   </body>
  65 </html>
  66 """
  67
  68
  69 def GetBlobStorage():
  70   """Get blob-storage from api-proxy stub map.
  71
  72   Returns:
  73     BlobStorage instance as registered with blobstore API in stub map.
  74   """
  75   return apiproxy_stub_map.apiproxy.GetStub('blobstore').storage
  76
  77
  78 def ParseRangeHeader(range_header):
  79   """Parse HTTP Range header.
  80
  81   Args:
  82     range_header: A str representing the value of a range header as retrived
  83       from Range or X-AppEngine-BlobRange.
  84
  85   Returns:
  86     Tuple (start, end):
  87       start: Start index of blob to retrieve.  May be negative index.
  88       end: None or end index.  End index is exclusive.
  89     (None, None) if there is a parse error.
  90   """
  91   if not range_header:
  92     return None, None
  93   try:
  94
  95     range_type, ranges = range_header.split('=', 1)
  96     if range_type != 'bytes':
  97       return None, None
  98     ranges = ranges.lstrip()
  99     if ',' in ranges:
 100       return None, None
 101     end = None
 102     if ranges.startswith('-'):
 103       start = int(ranges)
 104       if start == 0:
 105         return None, None
 106     else:
 107       split_range = ranges.split('-', 1)
 108       start = int(split_range[0])
 109       if len(split_range) == 2 and split_range[1].strip():
 110         end = int(split_range[1]) + 1
 111         if start > end:
 112           return None, None
 113     return start, end
 114   except ValueError:
 115     return None, None
 116
 117
 118 def _GetGoogleStorageFileMetadata(blob_key):
 119   """Retreive metadata about a GS blob from the blob_key.
 120
 121   Args:
 122     blob_key: The BlobKey of the blob.
 123
 124   Returns:
 125     Tuple (size, content_type, open_key):
 126       size: The size of the blob.
 127       content_type: The content type of the blob.
 128       open_key: The key used as an argument to BlobStorage to open the blob
 129         for reading.
 130     (None, None, None) if the blob metadata was not found.
 131   """
 132   try:
 133     gs_info = datastore.Get(
 134         datastore.Key.from_path(file_service_stub.GS_INFO_KIND,
 135                                 blob_key,
 136                                 namespace=''))
 137     return gs_info['size'], gs_info['content_type'], gs_info['storage_key']
 138   except datastore_errors.EntityNotFoundError:
 139     return None, None, None
 140
 141
 142 def _GetBlobstoreMetadata(blob_key):
 143   """Retreive metadata about a blobstore blob from the blob_key.
 144
 145   Args:
 146     blob_key: The BlobKey of the blob.
 147
 148   Returns:
 149     Tuple (size, content_type, open_key):
 150       size: The size of the blob.
 151       content_type: The content type of the blob.
 152       open_key: The key used as an argument to BlobStorage to open the blob
 153         for reading.
 154     (None, None, None) if the blob metadata was not found.
 155   """
 156   try:
 157     blob_info = datastore.Get(
 158         datastore.Key.from_path(blobstore.BLOB_INFO_KIND,
 159                                 blob_key,
 160                                 namespace=''))
 161     return blob_info['size'], blob_info['content_type'], blob_key
 162   except datastore_errors.EntityNotFoundError:
 163     return None, None, None
 164
 165
 166 def _GetBlobMetadata(blob_key):
 167   """Retrieve the metadata about a blob from the blob_key.
 168
 169   Args:
 170     blob_key: The BlobKey of the blob.
 171
 172   Returns:
 173     Tuple (size, content_type, open_key):
 174       size: The size of the blob.
 175       content_type: The content type of the blob.
 176       open_key: The key used as an argument to BlobStorage to open the blob
 177         for reading.
 178     (None, None, None) if the blob metadata was not found.
 179   """
 180   size, content_type, open_key = _GetGoogleStorageFileMetadata(blob_key)
 181   if size is None:
 182     size, content_type, open_key = _GetBlobstoreMetadata(blob_key)
 183   return size, content_type, open_key
 184
 185
 186 def _SetRangeRequestNotSatisfiable(response, blob_size):
 187   """Short circuit response and return 416 error.
 188
 189   Args:
 190     response: Response object to be rewritten.
 191     blob_size: The size of the blob.
 192   """
 193   response.status_code = 416
 194   response.status_message = 'Requested Range Not Satisfiable'
 195   response.body = cStringIO.StringIO('')
 196   response.headers['Content-Length'] = '0'
 197   response.headers['Content-Range'] = '*/%d' % blob_size
 198   del response.headers['Content-Type']
 199
 200
 201 def DownloadRewriter(response, request_headers):
 202   """Intercepts blob download key and rewrites response with large download.
 203
 204   Checks for the X-AppEngine-BlobKey header in the response.  If found, it will
 205   discard the body of the request and replace it with the blob content
 206   indicated.
 207
 208   If a valid blob is not found, it will send a 404 to the client.
 209
 210   If the application itself provides a content-type header, it will override
 211   the content-type stored in the action blob.
 212
 213   If blobstore.BLOB_RANGE_HEADER header is provided, blob will be partially
 214   served.  If Range is present, and not blobstore.BLOB_RANGE_HEADER, will use
 215   Range instead.
 216
 217   Args:
 218     response: Response object to be rewritten.
 219     request_headers: Original request headers.  Looks for 'Range' header to copy
 220       to response.
 221   """
 222   blob_key = response.headers.getheader(blobstore.BLOB_KEY_HEADER)
 223   if blob_key:
 224     del response.headers[blobstore.BLOB_KEY_HEADER]
 225
 226     blob_size, blob_content_type, blob_open_key = _GetBlobMetadata(blob_key)
 227
 228     range_header = response.headers.getheader(blobstore.BLOB_RANGE_HEADER)
 229     if range_header is not None:
 230       del response.headers[blobstore.BLOB_RANGE_HEADER]
 231     else:
 232       range_header = request_headers.getheader('Range')
 233
 234
 235
 236     if (blob_size is not None and blob_content_type is not None and
 237         response.status_code == 200):
 238       content_length = blob_size
 239       start = 0
 240       end = content_length
 241
 242       if range_header:
 243         start, end = ParseRangeHeader(range_header)
 244         if start is None:
 245           _SetRangeRequestNotSatisfiable(response, blob_size)
 246           return
 247         else:
 248           if start < 0:
 249             start = max(blob_size + start, 0)
 250           elif start >= blob_size:
 251             _SetRangeRequestNotSatisfiable(response, blob_size)
 252             return
 253           if end is not None:
 254             end = min(end, blob_size)
 255           else:
 256             end = blob_size
 257           content_length = min(end, blob_size) - start
 258           end = start + content_length
 259           response.status_code = 206
 260           response.status_message = 'Partial Content'
 261           response.headers['Content-Range'] = 'bytes %d-%d/%d' % (
 262               start, end - 1, blob_size)
 263
 264       blob_stream = GetBlobStorage().OpenBlob(blob_open_key)
 265       blob_stream.seek(start)
 266       response.body = cStringIO.StringIO(blob_stream.read(content_length))
 267       response.headers['Content-Length'] = str(content_length)
 268
 269       content_type = response.headers.getheader('Content-Type')
 270       if not content_type or content_type == AUTO_MIME_TYPE:
 271         response.headers['Content-Type'] = blob_content_type
 272       response.large_response = True
 273
 274     else:
 275
 276       if response.status_code != 200:
 277         logging.error('Blob-serving response with status %d, expected 200.',
 278                       response.status_code)
 279       else:
 280         logging.error('Could not find blob with key %s.', blob_key)
 281
 282       response.status_code = 500
 283       response.status_message = 'Internal Error'
 284       response.body = cStringIO.StringIO()
 285
 286       if response.headers.getheader('content-type'):
 287         del response.headers['content-type']
 288       response.headers['Content-Length'] = '0'
 289
 290
 291 def CreateUploadDispatcher(get_blob_storage=GetBlobStorage):
 292   """Function to create upload dispatcher.
 293
 294   Returns:
 295     New dispatcher capable of handling large blob uploads.
 296   """
 297
 298
 299   from google.appengine.tools import old_dev_appserver
 300
 301   class UploadDispatcher(old_dev_appserver.URLDispatcher):
 302     """Dispatcher that handles uploads."""
 303
 304     def __init__(self):
 305       """Constructor.
 306
 307       Args:
 308         blob_storage: A BlobStorage instance.
 309       """
 310       self.__cgi_handler = dev_appserver_upload.UploadCGIHandler(
 311           get_blob_storage())
 312
 313
 314
 315     def Dispatch(self,
 316                  request,
 317                  outfile,
 318                  base_env_dict=None):
 319       """Handle post dispatch.
 320
 321       This dispatcher will handle all uploaded files in the POST request, store
 322       the results in the blob-storage, close the upload session and transform
 323       the original request in to one where the uploaded files have external
 324       bodies.
 325
 326       Returns:
 327         New AppServerRequest indicating request forward to upload success
 328         handler.
 329       """
 330
 331       if base_env_dict['REQUEST_METHOD'] != 'POST':
 332         outfile.write('Status: 400\n\n')
 333         return
 334
 335
 336       upload_key = re.match(UPLOAD_URL_PATTERN, request.relative_url).group(1)
 337       try:
 338         upload_session = datastore.Get(upload_key)
 339       except datastore_errors.EntityNotFoundError:
 340         upload_session = None
 341
 342       if upload_session:
 343         success_path = upload_session['success_path']
 344         max_bytes_per_blob = upload_session['max_bytes_per_blob']
 345         max_bytes_total = upload_session['max_bytes_total']
 346         bucket_name = upload_session.get('gs_bucket_name', None)
 347
 348         upload_form = cgi.FieldStorage(fp=request.infile,
 349                                        headers=request.headers,
 350                                        environ=base_env_dict)
 351
 352         try:
 353
 354
 355           mime_message_string = self.__cgi_handler.GenerateMIMEMessageString(
 356               upload_form,
 357               max_bytes_per_blob=max_bytes_per_blob,
 358               max_bytes_total=max_bytes_total,
 359               bucket_name=bucket_name)
 360
 361           datastore.Delete(upload_session)
 362           self.current_session = upload_session
 363
 364
 365           header_end = mime_message_string.find('\n\n') + 1
 366           content_start = header_end + 1
 367           header_text = mime_message_string[:header_end].replace('\n', '\r\n')
 368           content_text = mime_message_string[content_start:].replace('\n',
 369                                                                      '\r\n')
 370
 371
 372           complete_headers = ('%s'
 373                               'Content-Length: %d\r\n'
 374                               '\r\n') % (header_text, len(content_text))
 375
 376           return old_dev_appserver.AppServerRequest(
 377               success_path,
 378               None,
 379               mimetools.Message(cStringIO.StringIO(complete_headers)),
 380               cStringIO.StringIO(content_text),
 381               force_admin=True)
 382         except dev_appserver_upload.InvalidMIMETypeFormatError:
 383           outfile.write('Status: 400\n\n')
 384         except dev_appserver_upload.UploadEntityTooLargeError:
 385           outfile.write('Status: 413\n\n')
 386           response = ERROR_RESPONSE_TEMPLATE % {
 387               'response_code': 413,
 388               'response_string': 'Request Entity Too Large',
 389               'response_text': 'Your client issued a request that was too '
 390               'large.'}
 391           outfile.write(response)
 392         except dev_appserver_upload.FilenameOrContentTypeTooLargeError, ex:
 393           outfile.write('Status: 400\n\n')
 394           response = ERROR_RESPONSE_TEMPLATE % {
 395               'response_code': 400,
 396               'response_string': 'Bad Request',
 397               'response_text': str(ex)}
 398           outfile.write(response)
 399       else:
 400         logging.error('Could not find session for %s', upload_key)
 401         outfile.write('Status: 404\n\n')
 402
 403
 404     def EndRedirect(self, dispatched_output, original_output):
 405       """Handle the end of upload complete notification.
 406
 407       Makes sure the application upload handler returned an appropriate status
 408       code.
 409       """
 410       response = old_dev_appserver.RewriteResponse(dispatched_output)
 411       logging.info('Upload handler returned %d', response.status_code)
 412       outfile = cStringIO.StringIO()
 413       outfile.write('Status: %s\n' % response.status_code)
 414
 415       if response.body and len(response.body.read()) > 0:
 416         response.body.seek(0)
 417         outfile.write(response.body.read())
 418       else:
 419         outfile.write(''.join(response.headers.headers))
 420
 421       outfile.seek(0)
 422       old_dev_appserver.URLDispatcher.EndRedirect(self,
 423                                                   outfile,
 424                                                   original_output)
 425
 426   return UploadDispatcher()