python/google/appengine/tools/devappserver2/blob_upload_test.py

   1 #!/usr/bin/env python
   2 #
   3 # Copyright 2007 Google Inc.
   4 #
   5 # Licensed under the Apache License, Version 2.0 (the "License");
   6 # you may not use this file except in compliance with the License.
   7 # You may obtain a copy of the License at
   8 #
   9 #     http://www.apache.org/licenses/LICENSE-2.0
  10 #
  11 # Unless required by applicable law or agreed to in writing, software
  12 # distributed under the License is distributed on an "AS IS" BASIS,
  13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14 # See the License for the specific language governing permissions and
  15 # limitations under the License.
  16 #
  17 """Tests for devappserver2.blob_upload."""
  18
  19
  20 import base64
  21 import cgi
  22 import cStringIO
  23 import datetime
  24 import email
  25 import email.message
  26 import hashlib
  27 import os
  28 import re
  29 import shutil
  30 import StringIO
  31 import tempfile
  32 import unittest
  33 import urlparse
  34 import wsgiref.util
  35
  36 import google
  37 import mox
  38 import webob.exc
  39
  40 from google.appengine.api import apiproxy_stub_map
  41 from google.appengine.api import datastore
  42 from google.appengine.api import datastore_errors
  43 from google.appengine.api import datastore_file_stub
  44 from google.appengine.api import namespace_manager
  45 from google.appengine.api import user_service_stub
  46 from google.appengine.api.blobstore import blobstore_stub
  47 from google.appengine.api.blobstore import file_blob_storage
  48 from google.appengine.ext import blobstore
  49 from google.appengine.tools.devappserver2 import blob_upload
  50 from google.appengine.tools.devappserver2 import constants
  51
  52 EXPECTED_GENERATED_CONTENT_TYPE = (
  53     'multipart/form-data; boundary="================1234=="')
  54 EXPECTED_GENERATED_MIME_MESSAGE = (
  55     """--================1234==
  56 Content-Type: message/external-body; blob-key="item1"; \
  57 access-type="X-AppEngine-BlobKey"
  58 Content-Disposition: form-data; name="field1"; filename="stuff.png"
  59
  60 Content-Type: image/png; a="b"; x="y"
  61 h2: v2
  62 Content-MD5: ODI2ZTgxNDJlNmJhYWJlOGFmNzc5ZjVmNDkwY2Y1ZjU=
  63 Content-Length: 5
  64 h1: v1
  65 X-AppEngine-Upload-Creation: 2008-11-12 10:40:00.000000
  66 Content-Disposition: form-data; name="field1"; filename="stuff.png"
  67
  68
  69 --================1234==
  70 Content-Type: message/external-body; blob-key="item2"; \
  71 access-type="X-AppEngine-BlobKey"
  72 Content-Disposition: form-data; name="field2"; filename="stuff.pdf"
  73
  74 Content-Type: application/pdf
  75 Content-Length: 5
  76 Content-MD5: MWMxYzk2ZmQyY2Y4MzMwZGIwYmZhOTM2Y2U4MmYzYjk=
  77 X-AppEngine-Upload-Creation: 2008-11-12 10:40:00.000000
  78 Content-Disposition: form-data; name="field2"; filename="stuff.pdf"
  79
  80
  81 --================1234==
  82 Content-Type: message/external-body; blob-key="item3"; \
  83 access-type="X-AppEngine-BlobKey"
  84 Content-Disposition: form-data; name="field2"; filename="stuff.txt"
  85
  86 Content-Type: text/plain
  87 Content-Length: 11
  88 Content-MD5: YmRjMDNkMGEyMTQwMTRlNjMyM2EyNGQzZDkzOTczNWY=
  89 X-AppEngine-Upload-Creation: 2008-11-12 10:40:00.000000
  90 Content-Disposition: form-data; name="field2"; filename="stuff.txt"
  91
  92
  93 --================1234==
  94 Content-Type: text/plain
  95 Content-Disposition: form-data; name="field3"
  96
  97 variable1
  98 --================1234==--""").replace('\n', '\r\n')
  99
 100 EXPECTED_GENERATED_CONTENT_TYPE_WITH_BUCKET = (
 101     'multipart/form-data; boundary="================1234=="')
 102 EXPECTED_GENERATED_MIME_MESSAGE_WITH_BUCKET = (
 103     """--================1234==
 104 Content-Type: message/external-body; blob-key="encoded_gs_file:\
 105 bXktdGVzdC1idWNrZXQvZmFrZS1leHBlY3RlZGtleQ=="; \
 106 access-type="X-AppEngine-BlobKey"
 107 Content-Disposition: form-data; name="field1"; filename="stuff.png"
 108
 109 Content-Type: image/png; a="b"; x="y"
 110 h2: v2
 111 Content-MD5: ODI2ZTgxNDJlNmJhYWJlOGFmNzc5ZjVmNDkwY2Y1ZjU=
 112 Content-Length: 5
 113 h1: v1
 114 X-AppEngine-Upload-Creation: 2008-11-12 10:40:00.000000
 115 X-AppEngine-Cloud-Storage-Object: /gs/my-test-bucket/fake-expectedkey
 116 Content-Disposition: form-data; name="field1"; filename="stuff.png"
 117
 118
 119 --================1234==--""").replace('\n', '\r\n')
 120
 121 EXPECTED_GENERATED_UTF8_CONTENT_TYPE = (
 122     'multipart/form-data; boundary="================1234=="')
 123 EXPECTED_GENERATED_UTF8_MIME_MESSAGE = (
 124     """--================1234==
 125 Content-Type: message/external-body; blob-key="item1"; \
 126 access-type="X-AppEngine-BlobKey"
 127 Content-Disposition: form-data; name="field1"; \
 128 filename="chinese_char_name_\xe6\xb1\x89.txt"
 129
 130 Content-Type: text/plain; a="b"; x="y"
 131 h2: v2
 132 Content-MD5: ODI2ZTgxNDJlNmJhYWJlOGFmNzc5ZjVmNDkwY2Y1ZjU=
 133 Content-Length: 5
 134 h1: v1
 135 X-AppEngine-Upload-Creation: 2008-11-12 10:40:00.000000
 136 Content-Disposition: form-data; name="field1"; \
 137 filename="chinese_char_name_\xe6\xb1\x89.txt"
 138
 139
 140 --================1234==--""").replace('\n', '\r\n')
 141
 142 EXPECTED_GENERATED_CONTENT_TYPE_NO_HEADERS = (
 143     'multipart/form-data; boundary="================1234=="')
 144 EXPECTED_GENERATED_MIME_MESSAGE_NO_HEADERS = (
 145     """--================1234==
 146 Content-Type: message/external-body; blob-key="item1"; \
 147 access-type="X-AppEngine-BlobKey"
 148 Content-Disposition: form-data; name="field1"; filename="file1"
 149
 150 Content-Type: application/octet-stream
 151 Content-Length: 5
 152 Content-MD5: ODI2ZTgxNDJlNmJhYWJlOGFmNzc5ZjVmNDkwY2Y1ZjU=
 153 X-AppEngine-Upload-Creation: 2008-11-12 10:40:00.000100
 154 Content-Disposition: form-data; name="field1"; filename="file1"
 155
 156
 157 --================1234==
 158 Content-Type: text/plain
 159 Content-Disposition: form-data; name="field2"
 160
 161 variable1
 162 --================1234==--""").replace('\n', '\r\n')
 163
 164 EXPECTED_GENERATED_CONTENT_TYPE_ZERO_LENGTH_BLOB = (
 165     'multipart/form-data; boundary="================1234=="')
 166 EXPECTED_GENERATED_MIME_MESSAGE_ZERO_LENGTH_BLOB = (
 167     """--================1234==
 168 Content-Type: message/external-body; blob-key="item1"; \
 169 access-type="X-AppEngine-BlobKey"
 170 Content-Disposition: form-data; name="field1"; filename="stuff.png"
 171
 172 Content-Type: image/png; a="b"; x="y"
 173 h2: v2
 174 Content-MD5: ODI2ZTgxNDJlNmJhYWJlOGFmNzc5ZjVmNDkwY2Y1ZjU=
 175 Content-Length: 5
 176 h1: v1
 177 X-AppEngine-Upload-Creation: 2008-11-12 10:40:00.000000
 178 Content-Disposition: form-data; name="field1"; filename="stuff.png"
 179
 180
 181 --================1234==
 182 Content-Type: message/external-body; blob-key="item2"; \
 183 access-type="X-AppEngine-BlobKey"
 184 Content-Disposition: form-data; name="field2"; filename="stuff.pdf"
 185
 186 Content-Type: application/pdf
 187 Content-Length: 0
 188 Content-MD5: ZDQxZDhjZDk4ZjAwYjIwNGU5ODAwOTk4ZWNmODQyN2U=
 189 X-AppEngine-Upload-Creation: 2008-11-12 10:40:00.000000
 190 Content-Disposition: form-data; name="field2"; filename="stuff.pdf"
 191
 192
 193 --================1234==--""").replace('\n', '\r\n')
 194
 195 EXPECTED_GENERATED_CONTENT_TYPE_NO_FILENAME = (
 196     'multipart/form-data; boundary="================1234=="')
 197 EXPECTED_GENERATED_MIME_MESSAGE_NO_FILENAME = (
 198     """--================1234==
 199 Content-Type: message/external-body; blob-key="item1"; \
 200 access-type="X-AppEngine-BlobKey"
 201 Content-Disposition: form-data; name="field1"; filename="stuff.png"
 202
 203 Content-Type: image/png; a="b"; x="y"
 204 h2: v2
 205 Content-MD5: ODI2ZTgxNDJlNmJhYWJlOGFmNzc5ZjVmNDkwY2Y1ZjU=
 206 Content-Length: 5
 207 h1: v1
 208 X-AppEngine-Upload-Creation: 2008-11-12 10:40:00.000000
 209 Content-Disposition: form-data; name="field1"; filename="stuff.png"
 210
 211
 212 --================1234==--""").replace('\n', '\r\n')
 213
 214 BAD_MIMES = ('/', 'image', 'image/', '/gif', 'app/monkey/banana')
 215
 216
 217 class FakeForm(dict):
 218   """Simple assignable object for emulating cgi.FieldStorage."""
 219
 220   def __init__(self, subforms=None, headers=None, **kwds):
 221     """Construct form from keywords."""
 222     super(FakeForm, self).__init__()
 223     self.update(subforms or {})
 224     self.headers = headers or email.Message.Message()
 225     for key, value in kwds.iteritems():
 226       setattr(self, key, value)
 227
 228
 229 class UploadTestBase(unittest.TestCase):
 230   """Base class for testing dev-appserver upload library."""
 231
 232   def setUp(self):
 233     """Configure test harness."""
 234     # Configure os.environ to make it look like the relevant parts of the
 235     # CGI environment that the stub relies on.
 236     self.original_environ = dict(os.environ)
 237     os.environ.update({
 238         'APPLICATION_ID': 'app',
 239         'SERVER_NAME': 'localhost',
 240         'SERVER_PORT': '8080',
 241         'AUTH_DOMAIN': 'abcxyz.com',
 242         'USER_EMAIL': 'user@abcxyz.com',
 243         })
 244
 245     # Set up mox.
 246     self.mox = mox.Mox()
 247
 248     # Use a fresh file datastore stub.
 249     self.tmpdir = tempfile.mkdtemp()
 250     self.datastore_file = os.path.join(self.tmpdir, 'datastore_v3')
 251     self.history_file = os.path.join(self.tmpdir, 'history')
 252     for filename in [self.datastore_file, self.history_file]:
 253       if os.access(filename, os.F_OK):
 254         os.remove(filename)
 255
 256     self.stub = datastore_file_stub.DatastoreFileStub(
 257         'app', self.datastore_file, self.history_file, use_atexit=False)
 258
 259     self.apiproxy = apiproxy_stub_map.APIProxyStubMap()
 260     apiproxy_stub_map.apiproxy = self.apiproxy
 261     apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', self.stub)
 262
 263   def tearDown(self):
 264     """Restore original environment."""
 265     os.environ = self.original_environ
 266     shutil.rmtree(self.tmpdir)
 267
 268   def assertMessageEqual(self, expected, actual):
 269     """Assert two strings representing messages are equal (equivalent).
 270
 271     This normalizes the headers in both arguments and then compares
 272     them using assertMultiLineEqual().
 273     """
 274     expected = self.normalize_header_lines(expected)
 275     actual = self.normalize_header_lines(actual)
 276     return self.assertMultiLineEqual(expected, actual)
 277
 278   def normalize_header_lines(self, message):
 279     """Normalize blocks of header lines in a message.
 280
 281     This sorts blocks of consecutive header lines and then for certain
 282     headers (Content-Type and -Disposition) sorts the parameter values.
 283     """
 284     lines = message.splitlines(True)
 285     # Normalize groups of header-like lines.
 286     output = []
 287     headers = []
 288     for line in lines:
 289       if re.match(r'^\S+: ', line):
 290         # It's a header line.  Maybe normalize the parameter values.
 291         line = self.normalize_header(line)
 292         headers.append(line)
 293       else:
 294         # Not a header.  Flush the list of headers.
 295         if headers:
 296           headers.sort()
 297           output.extend(headers)
 298           headers = []
 299         output.append(line)
 300     # Flush the final list of headers.
 301     if headers:
 302       headers.sort()
 303       output.extend(headers)
 304     # Put it all back together.
 305     return ''.join(output)
 306
 307   def normalize_header(self, line):
 308     """Normalize parameter values of Content-Type and -Disposition lines.
 309
 310     This changes e.g.
 311       Content-Type: foo/bar; name="a"; file="b"
 312     into
 313       Content-Type: foo/bar; file="b"; name="a"
 314
 315     It leaves other headers alone.
 316     """
 317     match = re.match(r'^(Content-(?:Type|Disposition): )(\S+; .*\S)(\s*)\Z',
 318                      line, re.IGNORECASE)
 319     if not match:
 320       return line
 321     value = match.group(2)
 322     value = self.normalize_parameter_order(value)
 323     return match.group(1) + value + match.group(3)
 324
 325   def normalize_parameter_order(self, value):
 326     """Normalize the parameter values of a header.
 327
 328     This changes e.g.
 329       foo/bar; name="a"; file="b"
 330     into
 331       foo/bar; file="b"; name="a"
 332
 333     Note that the text before the first ';' is unaffected.
 334     """
 335     parts = value.split('; ')
 336     if len(parts) > 2:
 337       value = parts[0] + '; ' + '; '.join(sorted(parts[1:]))
 338     return value
 339
 340
 341 class GenerateBlobKeyTest(UploadTestBase):
 342   """Tests the GenerateBlobKey function."""
 343
 344   def check_key(self, blob_key, expected_time, expected_random):
 345     """Check that blob_key decodes to expected value.
 346
 347     Args:
 348       blob_key: Blob key that was actually generated.
 349       expected_time: Time stamp that is expected to be in the md5 digest.
 350       expected_random: Random number that is expected to be in the md5 digest.
 351     """
 352     if blob_key is None:
 353       self.fail('Generated blob-key is None.')
 354     digester = hashlib.md5()
 355     digester.update(str(expected_time))
 356     digester.update(str(expected_random))
 357     actual_digest = base64.urlsafe_b64decode(blob_key)
 358     self.assertEquals(digester.digest(), actual_digest)
 359
 360   def test_generate_key(self):
 361     """Basic test of key generation."""
 362     time_func = self.mox.CreateMockAnything()
 363     random_func = self.mox.CreateMockAnything()
 364
 365     time_func().AndReturn(10)
 366     random_func().AndReturn(20)
 367
 368     self.mox.ReplayAll()
 369
 370     key = blob_upload._generate_blob_key(time_func, random_func)
 371     self.check_key(key, 10, 20)
 372
 373     self.mox.VerifyAll()
 374
 375   def test_generate_key_with_conflict(self):
 376     """Test what happens when there is conflict in key generation."""
 377     time_func = self.mox.CreateMockAnything()
 378     random_func = self.mox.CreateMockAnything()
 379
 380     time_func().AndReturn(10)
 381     random_func().AndReturn(20)
 382
 383     time_func().AndReturn(10)
 384     random_func().AndReturn(30)
 385
 386     time_func().AndReturn(10)
 387     random_func().AndReturn(20)
 388     random_func().AndReturn(30)
 389     random_func().AndReturn(40)
 390
 391     self.mox.ReplayAll()
 392
 393     # Create a pair of conflicting records.
 394     entity = datastore.Entity(
 395         blobstore.BLOB_INFO_KIND,
 396         name=str(blob_upload._generate_blob_key(time_func, random_func)),
 397         namespace='')
 398     datastore.Put(entity)
 399     entity = datastore.Entity(
 400         blobstore.BLOB_INFO_KIND,
 401         name=str(blob_upload._generate_blob_key(time_func, random_func)),
 402         namespace='')
 403     datastore.Put(entity)
 404
 405     key = blob_upload._generate_blob_key(time_func, random_func)
 406     self.check_key(key, 10, 40)
 407
 408     self.mox.VerifyAll()
 409
 410   def test_too_many_conflicts(self):
 411     """Test what happens when there are too many conflicts in key generation."""
 412     time_func = self.mox.CreateMockAnything()
 413     random_func = self.mox.CreateMockAnything()
 414
 415     # Create first set of keys
 416     for i in range(10):
 417       time_func().AndReturn(10)
 418       random_func().AndReturn(10 + i)
 419
 420     # Try to create duplicate keys
 421     time_func().AndReturn(10)
 422     for i in range(10):
 423       random_func().AndReturn(10 + i)
 424
 425     self.mox.ReplayAll()
 426
 427     # Create a pair of conflicting records.
 428     for i in range(10):
 429       entity = datastore.Entity(
 430           blobstore.BLOB_INFO_KIND,
 431           name=str(blob_upload._generate_blob_key(time_func, random_func)),
 432           namespace='')
 433       datastore.Put(entity)
 434     self.assertRaises(blob_upload._TooManyConflictsError,
 435                       blob_upload._generate_blob_key,
 436                       time_func, random_func)
 437
 438     self.mox.VerifyAll()
 439
 440
 441 class GenerateBlobKeyTestNamespace(GenerateBlobKeyTest):
 442   """Executes all of the superclass tests but with a namespace set."""
 443
 444   def setUp(self):
 445     """Setup for namespaces test."""
 446     super(GenerateBlobKeyTestNamespace, self).setUp()
 447     # Set the namespace. Blobstore should ignore this.
 448     namespace_manager.set_namespace('abc')
 449
 450
 451 class UploadHandlerUnitTest(UploadTestBase):
 452   """Test the UploadHandler class's individual methods."""
 453
 454   def setUp(self):
 455     """Set up additional parts of the test framework."""
 456     UploadTestBase.setUp(self)
 457
 458     # Create a phoney blob-generation method for predictable key generation.
 459     self.generate_blob_key = self.mox.CreateMockAnything()
 460     # Create a mock now function for predictable timestamp generation.
 461     self.now = self.mox.CreateMockAnything()
 462
 463     # Create blob-storage to be used in tests.
 464     self.blob_storage_path = os.path.join(self.tmpdir, 'blobstore')
 465     self.storage = file_blob_storage.FileBlobStorage(
 466         self.blob_storage_path,
 467         os.environ['APPLICATION_ID'])
 468
 469     def forward_app(unused_environ, unused_start_response):
 470       raise Exception('Unexpected call to forward_app')
 471
 472     def get_storage():
 473       return self.storage
 474
 475     # Create handler for testing.
 476     self.handler = blob_upload.Application(
 477         forward_app, get_storage, self.generate_blob_key, self.now)
 478
 479   def execute_blob_test(self, blob_content, expected_result,
 480                         base64_encoding=False):
 481     """Execute a basic blob insertion."""
 482     expected_key = blobstore.BlobKey('expectedkey')
 483     expected_creation = datetime.datetime(2008, 11, 12)
 484
 485     self.generate_blob_key().AndReturn(expected_key)
 486
 487     self.mox.ReplayAll()
 488     content_type, blob_file, filename = self.handler._preprocess_data(
 489         'image/png; a="b"; m="n"',
 490         StringIO.StringIO(blob_content),
 491         'stuff.png',
 492         base64_encoding)
 493     self.handler.store_blob(content_type=content_type,
 494                             filename=filename,
 495                             md5_hash=hashlib.md5(),
 496                             blob_file=blob_file,
 497                             creation=expected_creation)
 498
 499     self.assertEquals(expected_result,
 500                       self.storage.OpenBlob(expected_key).read())
 501
 502     blob_info = blobstore.get(expected_key)
 503     self.assertFalse(blob_info is None)
 504     self.assertEquals(('image/png', {'a': 'b', 'm': 'n'}),
 505                       cgi.parse_header(blob_info.content_type))
 506     self.assertEquals(expected_creation, blob_info.creation)
 507     self.assertEquals('stuff.png', blob_info.filename)
 508     self.assertEquals(len(expected_result), blob_info.size)
 509
 510     self.mox.VerifyAll()
 511
 512   def test_store_blob(self):
 513     """Test blob creation."""
 514     self.execute_blob_test('blob content', 'blob content')
 515
 516   def test_store_and_build_forward_message(self):
 517     """Test the high-level method to store a blob and build a MIME message."""
 518     self.generate_blob_key().AndReturn(blobstore.BlobKey('item1'))
 519     self.generate_blob_key().AndReturn(blobstore.BlobKey('item2'))
 520     self.generate_blob_key().AndReturn(blobstore.BlobKey('item3'))
 521
 522     self.now().AndReturn(datetime.datetime(2008, 11, 12, 10, 40))
 523
 524     self.mox.ReplayAll()
 525
 526     form = FakeForm({
 527         'field1': FakeForm(name='field1',
 528                            file=StringIO.StringIO('file1'),
 529                            type='image/png',
 530                            type_options={'a': 'b', 'x': 'y'},
 531                            filename='stuff.png',
 532                            headers={'h1': 'v1',
 533                                     'h2': 'v2',
 534                                    }),
 535         'field2': [FakeForm(name='field2',
 536                             file=StringIO.StringIO('file2'),
 537                             type='application/pdf',
 538                             type_options={},
 539                             filename='stuff.pdf',
 540                             headers={}),
 541                    FakeForm(name='field2',
 542                             file=StringIO.StringIO('file3 extra'),
 543                             type='text/plain',
 544                             type_options={},
 545                             filename='stuff.txt',
 546                             headers={}),
 547                   ],
 548         'field3': FakeForm(name='field3',
 549                            value='variable1',
 550                            type='text/plain',
 551                            type_options={},
 552                            filename=None),
 553         })
 554
 555     content_type, content_text = self.handler.store_and_build_forward_message(
 556         form, '================1234==')
 557
 558     self.mox.VerifyAll()
 559
 560     self.assertEqual(EXPECTED_GENERATED_CONTENT_TYPE, content_type)
 561     self.assertMessageEqual(EXPECTED_GENERATED_MIME_MESSAGE, content_text)
 562
 563     blob1 = blobstore.get('item1')
 564     self.assertEquals('stuff.png', blob1.filename)
 565     self.assertEquals(('image/png', {'a': 'b', 'x': 'y'}),
 566                       cgi.parse_header(blob1.content_type))
 567
 568     blob2 = blobstore.get('item2')
 569     self.assertEquals('stuff.pdf', blob2.filename)
 570     self.assertEquals('application/pdf', blob2.content_type)
 571
 572     blob3 = blobstore.get('item3')
 573     self.assertEquals('stuff.txt', blob3.filename)
 574     self.assertEquals('text/plain', blob3.content_type)
 575
 576   def test_store_and_build_forward_message_with_gs_bucket(self):
 577     """Test the high-level method to store a blob and build a MIME message."""
 578     self.now().AndReturn(datetime.datetime(2008, 11, 12, 10, 40))
 579     expected_key = blobstore.BlobKey('expectedkey')
 580     self.generate_blob_key().AndReturn(expected_key)
 581
 582     self.mox.ReplayAll()
 583
 584     form = FakeForm({
 585         'field1': FakeForm(name='field1',
 586                            file=StringIO.StringIO('file1'),
 587                            type='image/png',
 588                            type_options={'a': 'b', 'x': 'y'},
 589                            filename='stuff.png',
 590                            headers={'h1': 'v1',
 591                                     'h2': 'v2',
 592                                    }),
 593         })
 594
 595     content_type, content_text = self.handler.store_and_build_forward_message(
 596         form, '================1234==', bucket_name='my-test-bucket')
 597
 598     self.mox.VerifyAll()
 599
 600     self.assertEqual(EXPECTED_GENERATED_CONTENT_TYPE_WITH_BUCKET, content_type)
 601     self.assertMessageEqual(EXPECTED_GENERATED_MIME_MESSAGE_WITH_BUCKET,
 602                             content_text)
 603     blobkey = ('encoded_gs_file:bXktdGVzdC1idWNrZXQvZmFrZS1leHBlY3RlZGtleQ==')
 604     blobkey = blobstore_stub.BlobstoreServiceStub.ToDatastoreBlobKey(blobkey)
 605     blob1 = datastore.Get(blobkey)
 606     self.assertTrue('my-test-bucket' in blob1['filename'])
 607
 608   def test_store_and_build_forward_message_utf8_values(self):
 609     """Test store and build message method with UTF-8 values."""
 610     self.generate_blob_key().AndReturn(blobstore.BlobKey('item1'))
 611
 612     self.now().AndReturn(datetime.datetime(2008, 11, 12, 10, 40))
 613
 614     self.mox.ReplayAll()
 615
 616     form = FakeForm({
 617         'field1': FakeForm(name='field1',
 618                            file=StringIO.StringIO('file1'),
 619                            type='text/plain',
 620                            type_options={'a': 'b', 'x': 'y'},
 621                            filename='chinese_char_name_\xe6\xb1\x89.txt',
 622                            headers={'h1': 'v1',
 623                                     'h2': 'v2',
 624                                    }),
 625         })
 626
 627     content_type, content_text = self.handler.store_and_build_forward_message(
 628         form, '================1234==')
 629
 630     self.mox.VerifyAll()
 631
 632     self.assertEqual(EXPECTED_GENERATED_UTF8_CONTENT_TYPE, content_type)
 633     self.assertMessageEqual(EXPECTED_GENERATED_UTF8_MIME_MESSAGE,
 634                             content_text)
 635
 636     blob1 = blobstore.get('item1')
 637     self.assertEquals(u'chinese_char_name_\u6c49.txt', blob1.filename)
 638
 639   def test_store_and_build_forward_message_latin1_values(self):
 640     """Test store and build message method with Latin-1 values."""
 641     # There is a special exception class for this case. This is designed to
 642     # emulate production, which currently fails silently. See b/6722082.
 643     self.now().AndReturn(datetime.datetime(2008, 11, 12, 10, 40))
 644
 645     self.mox.ReplayAll()
 646
 647     form = FakeForm({
 648         'field1': FakeForm(name='field1',
 649                            file=StringIO.StringIO('file1'),
 650                            type='text/plain',
 651                            type_options={'a': 'b', 'x': 'y'},
 652                            filename='german_char_name_f\xfc\xdfe.txt',
 653                            headers={'h1': 'v1',
 654                                     'h2': 'v2',
 655                                    }),
 656         })
 657
 658     self.assertRaises(blob_upload._InvalidMetadataError,
 659                       self.handler.store_and_build_forward_message, form,
 660                       '================1234==')
 661
 662     self.mox.VerifyAll()
 663
 664     blob1 = blobstore.get('item1')
 665     self.assertIsNone(blob1)
 666
 667   def test_store_and_build_forward_message_no_headers(self):
 668     """Test default header generation when no headers are provided."""
 669     self.generate_blob_key().AndReturn(blobstore.BlobKey('item1'))
 670
 671     self.now().AndReturn(datetime.datetime(2008, 11, 12, 10, 40, 0, 100))
 672
 673     self.mox.ReplayAll()
 674
 675     form = FakeForm({'field1': FakeForm(name='field1',
 676                                         file=StringIO.StringIO('file1'),
 677                                         type=None,
 678                                         type_options={},
 679                                         filename='file1',
 680                                         headers={}),
 681                      'field2': FakeForm(name='field2',
 682                                         value='variable1',
 683                                         type=None,
 684                                         type_options={},
 685                                         filename=None,
 686                                         headers={}),
 687                     })
 688
 689     content_type, content_text = self.handler.store_and_build_forward_message(
 690         form, '================1234==')
 691
 692     self.mox.VerifyAll()
 693
 694     self.assertEqual(EXPECTED_GENERATED_CONTENT_TYPE_NO_HEADERS, content_type)
 695     self.assertMessageEqual(EXPECTED_GENERATED_MIME_MESSAGE_NO_HEADERS,
 696                             content_text)
 697
 698   def test_store_and_build_forward_message_zero_length_blob(self):
 699     """Test upload with a zero length blob."""
 700     self.generate_blob_key().AndReturn(blobstore.BlobKey('item1'))
 701     self.generate_blob_key().AndReturn(blobstore.BlobKey('item2'))
 702
 703     self.now().AndReturn(datetime.datetime(2008, 11, 12, 10, 40))
 704
 705     self.mox.ReplayAll()
 706
 707     form = FakeForm({
 708         'field1': FakeForm(name='field1',
 709                            file=StringIO.StringIO('file1'),
 710                            type='image/png',
 711                            type_options={'a': 'b', 'x': 'y'},
 712                            filename='stuff.png',
 713                            headers={'h1': 'v1',
 714                                     'h2': 'v2',
 715                                    }),
 716         'field2': FakeForm(name='field2',
 717                            file=StringIO.StringIO(''),
 718                            type='application/pdf',
 719                            type_options={},
 720                            filename='stuff.pdf',
 721                            headers={}),
 722         })
 723
 724     content_type, content_text = self.handler.store_and_build_forward_message(
 725         form, '================1234==')
 726
 727     self.mox.VerifyAll()
 728
 729     self.assertEqual(EXPECTED_GENERATED_CONTENT_TYPE_ZERO_LENGTH_BLOB,
 730                      content_type)
 731     self.assertMessageEqual(EXPECTED_GENERATED_MIME_MESSAGE_ZERO_LENGTH_BLOB,
 732                             content_text)
 733
 734     blob1 = blobstore.get('item1')
 735     self.assertEquals('stuff.png', blob1.filename)
 736
 737     blob2 = blobstore.get('item2')
 738     self.assertEquals('stuff.pdf', blob2.filename)
 739
 740   def test_store_and_build_forward_message_no_filename(self):
 741     """Test upload with no filename in content disposition."""
 742     self.generate_blob_key().AndReturn(blobstore.BlobKey('item1'))
 743
 744     self.now().AndReturn(datetime.datetime(2008, 11, 12, 10, 40))
 745
 746     self.mox.ReplayAll()
 747
 748     form = FakeForm({
 749         'field1': FakeForm(name='field1',
 750                            file=StringIO.StringIO('file1'),
 751                            type='image/png',
 752                            type_options={'a': 'b', 'x': 'y'},
 753                            filename='stuff.png',
 754                            headers={'h1': 'v1',
 755                                     'h2': 'v2',
 756                                    }),
 757         'field2': FakeForm(name='field2',
 758                            file=StringIO.StringIO(''),
 759                            type='application/pdf',
 760                            type_options={},
 761                            filename='',
 762                            headers={}),
 763         })
 764
 765     content_type, content_text = self.handler.store_and_build_forward_message(
 766         form, '================1234==')
 767
 768     self.mox.VerifyAll()
 769
 770     self.assertEqual(EXPECTED_GENERATED_CONTENT_TYPE_NO_FILENAME, content_type)
 771     self.assertMessageEqual(EXPECTED_GENERATED_MIME_MESSAGE_NO_FILENAME,
 772                             content_text)
 773
 774     blob1 = blobstore.get('item1')
 775     self.assertEquals('stuff.png', blob1.filename)
 776
 777     self.assertEquals(None, blobstore.get('item2'))
 778
 779   def test_store_and_build_forward_message_bad_mimes(self):
 780     """Test upload with no headers provided."""
 781     for unused_mime in range(len(BAD_MIMES)):
 782       # Should not require actual time value upon failure.
 783       self.now()
 784
 785     self.mox.ReplayAll()
 786
 787     for mime_type in BAD_MIMES:
 788       form = FakeForm({'field1': FakeForm(name='field1',
 789                                           file=StringIO.StringIO('file1'),
 790                                           type=mime_type,
 791                                           type_options={},
 792                                           filename='file',
 793                                           headers={}),
 794                       })
 795
 796       self.assertRaisesRegexp(
 797           webob.exc.HTTPClientError,
 798           'Incorrectly formatted MIME type: %s' % mime_type,
 799           self.handler.store_and_build_forward_message,
 800           form,
 801           '================1234==')
 802
 803     self.mox.VerifyAll()
 804
 805   def test_store_and_build_forward_message_max_blob_size_exceeded(self):
 806     """Test upload with a blob larger than the maximum blob size."""
 807     self.generate_blob_key().AndReturn(blobstore.BlobKey('item1'))
 808
 809     self.now().AndReturn(datetime.datetime(2008, 11, 12, 10, 40))
 810
 811     self.mox.ReplayAll()
 812
 813     form = FakeForm({
 814         'field1': FakeForm(name='field1',
 815                            file=StringIO.StringIO('a'),
 816                            type='image/png',
 817                            type_options={'a': 'b', 'x': 'y'},
 818                            filename='stuff.png',
 819                            headers={'h1': 'v1',
 820                                     'h2': 'v2',
 821                                    }),
 822         'field2': FakeForm(name='field2',
 823                            file=StringIO.StringIO('longerfile'),
 824                            type='application/pdf',
 825                            type_options={},
 826                            filename='stuff.pdf',
 827                            headers={}),
 828         })
 829
 830     self.assertRaises(webob.exc.HTTPRequestEntityTooLarge,
 831                       self.handler.store_and_build_forward_message,
 832                       form, '================1234==', max_bytes_per_blob=2)
 833
 834     self.mox.VerifyAll()
 835
 836     blob1 = blobstore.get('item1')
 837     self.assertIsNone(blob1)
 838
 839   def test_store_and_build_forward_message_total_size_exceeded(self):
 840     """Test upload with all blobs larger than the total allowed size."""
 841     self.generate_blob_key().AndReturn(blobstore.BlobKey('item1'))
 842
 843     self.now().AndReturn(datetime.datetime(2008, 11, 12, 10, 40))
 844
 845     self.mox.ReplayAll()
 846
 847     form = FakeForm({
 848         'field1': FakeForm(name='field1',
 849                            file=StringIO.StringIO('a'),
 850                            type='image/png',
 851                            type_options={'a': 'b', 'x': 'y'},
 852                            filename='stuff.png',
 853                            headers={'h1': 'v1',
 854                                     'h2': 'v2',
 855                                    }),
 856         'field2': FakeForm(name='field2',
 857                            file=StringIO.StringIO('longerfile'),
 858                            type='application/pdf',
 859                            type_options={},
 860                            filename='stuff.pdf',
 861                            headers={}),
 862         })
 863
 864     self.assertRaises(webob.exc.HTTPRequestEntityTooLarge,
 865                       self.handler.store_and_build_forward_message,
 866                       form, '================1234==', max_bytes_total=3)
 867
 868     self.mox.VerifyAll()
 869
 870     blob1 = blobstore.get('item1')
 871     self.assertIsNone(blob1)
 872
 873   def test_store_blob_base64(self):
 874     """Test blob creation with a base-64-encoded body."""
 875     expected_result = 'This is the blob content.'
 876     self.execute_blob_test(base64.urlsafe_b64encode(expected_result),
 877                            expected_result,
 878                            base64_encoding=True)
 879
 880   def test_filename_too_large(self):
 881     """Test that exception is raised if the filename is too large."""
 882     self.now().AndReturn(datetime.datetime(2008, 11, 12, 10, 40))
 883     self.mox.ReplayAll()
 884
 885     filename = 'a' * blob_upload._MAX_STRING_NAME_LENGTH + '.txt'
 886
 887     form = FakeForm({
 888         'field1': FakeForm(name='field1',
 889                            file=StringIO.StringIO('a'),
 890                            type='image/png',
 891                            type_options={'a': 'b', 'x': 'y'},
 892                            filename=filename,
 893                            headers={}),
 894         })
 895
 896     self.assertRaisesRegexp(
 897         webob.exc.HTTPClientError,
 898         'The filename exceeds the maximum allowed length of 500.',
 899         self.handler.store_and_build_forward_message,
 900         form, '================1234==')
 901
 902     self.mox.VerifyAll()
 903
 904   def test_content_type_too_large(self):
 905     """Test that exception is raised if the content-type is too large."""
 906     self.now().AndReturn(datetime.datetime(2008, 11, 12, 10, 40))
 907     self.mox.ReplayAll()
 908
 909     content_type = 'text/' + 'a' * blob_upload._MAX_STRING_NAME_LENGTH
 910
 911     form = FakeForm({
 912         'field1': FakeForm(name='field1',
 913                            file=StringIO.StringIO('a'),
 914                            type=content_type,
 915                            type_options={'a': 'b', 'x': 'y'},
 916                            filename='foobar.txt',
 917                            headers={}),
 918         })
 919
 920     self.assertRaisesRegexp(
 921         webob.exc.HTTPClientError,
 922         'The Content-Type exceeds the maximum allowed length of 500.',
 923         self.handler.store_and_build_forward_message,
 924         form, '================1234==')
 925
 926     self.mox.VerifyAll()
 927
 928
 929 class UploadHandlerUnitTestNamespace(UploadHandlerUnitTest):
 930   """Executes all of the superclass tests but with a namespace set."""
 931
 932   def setUp(self):
 933     """Setup for namespaces test."""
 934     super(UploadHandlerUnitTestNamespace, self).setUp()
 935     # Set the namespace. Blobstore should ignore this.
 936     namespace_manager.set_namespace('abc')
 937
 938
 939 class UploadHandlerWSGITest(UploadTestBase):
 940   """Test the upload handler as a whole, by making WSGI requests."""
 941
 942   def setUp(self):
 943     """Set up test framework."""
 944     # Set up environment for Blobstore.
 945     self.original_environ = dict(os.environ)
 946     os.environ.update({
 947         'APPLICATION_ID': 'app',
 948         'USER_EMAIL': 'nobody@nowhere.com',
 949         'SERVER_NAME': 'localhost',
 950         'SERVER_PORT': '8080',
 951         })
 952     self.environ = {}
 953     wsgiref.util.setup_testing_defaults(self.environ)
 954     self.environ['REQUEST_METHOD'] = 'POST'
 955
 956     # Set up user stub.
 957     self.user_stub = user_service_stub.UserServiceStub()
 958
 959     self.tmpdir = tempfile.mkdtemp()
 960
 961     ## Set up testing blobstore files.
 962     storage_directory = os.path.join(self.tmpdir, 'blobstore')
 963     self.blob_storage = file_blob_storage.FileBlobStorage(storage_directory,
 964                                                           'appid1')
 965     self.blobstore_stub = blobstore_stub.BlobstoreServiceStub(self.blob_storage)
 966
 967     # Use a fresh file datastore stub.
 968     self.datastore_file = os.path.join(self.tmpdir, 'datastore_v3')
 969     self.history_file = os.path.join(self.tmpdir, 'history')
 970     for filename in [self.datastore_file, self.history_file]:
 971       if os.access(filename, os.F_OK):
 972         os.remove(filename)
 973
 974     self.datastore_stub = datastore_file_stub.DatastoreFileStub(
 975         'app', self.datastore_file, self.history_file, use_atexit=False)
 976
 977     self.apiproxy = apiproxy_stub_map.APIProxyStubMap()
 978     apiproxy_stub_map.apiproxy = self.apiproxy
 979     apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', self.datastore_stub)
 980     apiproxy_stub_map.apiproxy.RegisterStub('blobstore', self.blobstore_stub)
 981     apiproxy_stub_map.apiproxy.RegisterStub('user', self.user_stub)
 982
 983     # Keep values given to forward_app.
 984     self.forward_request_dict = {}
 985
 986     def forward_app(environ, start_response):
 987       self.forward_request_dict['environ'] = environ
 988       self.forward_request_dict['body'] = environ['wsgi.input'].read()
 989       # Return a dummy body
 990       start_response('200 OK', [('CONTENT_TYPE', 'text/plain')])
 991       return ['Forwarded successfully.']
 992
 993     self.dispatcher = blob_upload.Application(forward_app)
 994
 995   def tearDown(self):
 996     os.environ = self.original_environ
 997     shutil.rmtree(self.tmpdir)
 998
 999   def run_dispatcher(self, request_body=''):
1000     """Runs self.dispatcher and returns the response.
1001
1002     self.environ should already be initialised with the WSGI environment,
1003     including the HTTP_* headers.
1004
1005     Args:
1006       request_body: String containing the body of the request.
1007
1008     Returns:
1009       (status, headers, response_body, forward_environ, forward_body), where:
1010         status is the response status string,
1011         headers is a dict containing the response headers (with lowercase
1012           names),
1013         response_body is a string containing the response body,
1014         forward_environ is the WSGI environ passed to the forwarded request, or
1015           None if the forward application was not called,
1016         forward_body is the request body passed to the forwarded request, or
1017           None if the forward application was not called.
1018
1019     Raises:
1020       AssertionError: start_response was not called.
1021       Exception: The WSGI application returned an exception.
1022     """
1023
1024     response_dict = {}
1025     state_dict = {
1026         'start_response_already_called': False,
1027         'headers_already_sent': False,
1028     }
1029
1030     self.environ['wsgi.input'] = cStringIO.StringIO(request_body)
1031
1032     body = cStringIO.StringIO()
1033
1034     def write_body(text):
1035       if not text:
1036         return
1037       assert state_dict['start_response_already_called']
1038       body.write(text)
1039       state_dict['headers_already_sent'] = True
1040
1041     def start_response(status, response_headers, exc_info=None):
1042       if exc_info is None:
1043         assert not state_dict['start_response_already_called']
1044       if state_dict['headers_already_sent']:
1045         raise exc_info[0], exc_info[1], exc_info[2]
1046       state_dict['start_response_already_called'] = True
1047       response_dict['status'] = status
1048       response_dict['headers'] = dict((k.lower(), v) for (k, v) in
1049                                       response_headers)
1050       return write_body
1051
1052     self.forward_request_dict['environ'] = None
1053     self.forward_request_dict['body'] = None
1054
1055     for s in self.dispatcher(self.environ, start_response):
1056       write_body(s)
1057
1058     if 'status' not in response_dict:
1059       self.fail('start_response was not called')
1060
1061     return (response_dict['status'], response_dict['headers'], body.getvalue(),
1062             self.forward_request_dict['environ'],
1063             self.forward_request_dict['body'])
1064
1065   def _run_test_success(self, upload_data, upload_url):
1066     """Basic dispatcher request flow."""
1067     request_path = urlparse.urlparse(upload_url)[2]
1068
1069     # Get session key from upload url.
1070     session_key = upload_url.split('/')[-1]
1071
1072     self.environ['PATH_INFO'] = request_path
1073     self.environ['CONTENT_TYPE'] = (
1074         'multipart/form-data; boundary="================1234=="')
1075     status, _, response_body, forward_environ, forward_body = (
1076         self.run_dispatcher(upload_data))
1077
1078     self.assertEquals('200 OK', status)
1079     self.assertEquals('Forwarded successfully.', response_body)
1080
1081     self.assertNotEquals(None, forward_environ)
1082
1083     # These must NOT be unicode strings.
1084     self.assertIsInstance(forward_environ['PATH_INFO'], str)
1085     if 'QUERY_STRING' in forward_environ:
1086       self.assertIsInstance(forward_environ['QUERY_STRING'], str)
1087     self.assertRegexpMatches(forward_environ['CONTENT_TYPE'],
1088                              r'multipart/form-data; boundary="[^"]+"')
1089     self.assertEquals(len(forward_body), int(forward_environ['CONTENT_LENGTH']))
1090     self.assertIn(constants.FAKE_IS_ADMIN_HEADER, forward_environ)
1091     self.assertEquals('1', forward_environ[constants.FAKE_IS_ADMIN_HEADER])
1092
1093     new_request = email.message_from_string(
1094         'Content-Type: %s\n\n%s' % (forward_environ['CONTENT_TYPE'],
1095                                     forward_body))
1096     (upload,) = new_request.get_payload()
1097     self.assertEquals('message/external-body', upload.get_content_type())
1098
1099     message = email.message.Message()
1100     message.add_header('Content-Type', upload['Content-Type'])
1101     blob_key = message.get_param('blob-key')
1102     blob_contents = blobstore.BlobReader(blob_key).read()
1103     self.assertEquals('value', blob_contents)
1104
1105     self.assertRaises(datastore_errors.EntityNotFoundError,
1106                       datastore.Get,
1107                       session_key)
1108
1109     return upload, forward_environ, forward_body
1110
1111   def test_success(self):
1112     """Basic dispatcher request flow."""
1113     # Create upload.
1114     upload_data = (
1115         """--================1234==
1116 Content-Type: text/plain
1117 MIME-Version: 1.0
1118 Content-Disposition: form-data; name="field1"; filename="stuff.txt"
1119
1120 value
1121 --================1234==--""")
1122
1123     upload_url = blobstore.create_upload_url('/success?foo=bar')
1124
1125     upload, forward_environ, _ = self._run_test_success(
1126         upload_data, upload_url)
1127
1128     self.assertEquals('/success', forward_environ['PATH_INFO'])
1129     self.assertEquals('foo=bar', forward_environ['QUERY_STRING'])
1130     self.assertEquals(
1131         ('form-data', {'filename': 'stuff.txt', 'name': 'field1'}),
1132         cgi.parse_header(upload['content-disposition']))
1133
1134   def test_success_with_bucket(self):
1135     """Basic dispatcher request flow."""
1136     # Create upload.
1137     upload_data = (
1138         """--================1234==
1139 Content-Type: text/plain
1140 MIME-Version: 1.0
1141 Content-Disposition: form-data; name="field1"; filename="stuff.txt"
1142
1143 value
1144 --================1234==--""")
1145
1146     upload_url = blobstore.create_upload_url('/success?foo=bar',
1147                                              gs_bucket_name='my_test_bucket')
1148
1149     upload, forward_environ, forward_body = self._run_test_success(
1150         upload_data, upload_url)
1151
1152     self.assertEquals('/success', forward_environ['PATH_INFO'])
1153     self.assertEquals('foo=bar', forward_environ['QUERY_STRING'])
1154     self.assertEquals(
1155         ('form-data', {'filename': 'stuff.txt', 'name': 'field1'}),
1156         cgi.parse_header(upload['content-disposition']))
1157     self.assertIn('X-AppEngine-Cloud-Storage-Object: /gs/%s' % 'my_test_bucket',
1158                   forward_body)
1159
1160   def test_success_full_success_url(self):
1161     """Request flow with a success url containing protocol, host and port."""
1162     # Create upload.
1163     upload_data = (
1164         """--================1234==
1165 Content-Type: text/plain
1166 MIME-Version: 1.0
1167 Content-Disposition: form-data; name="field1"; filename="stuff.txt"
1168
1169 value
1170 --================1234==--""")
1171
1172     # The scheme, host and port should all be ignored.
1173     upload_url = blobstore.create_upload_url(
1174         'https://example.com:1234/success?foo=bar')
1175
1176     upload, forward_environ, _ = self._run_test_success(
1177         upload_data, upload_url)
1178
1179     self.assertEquals('/success', forward_environ['PATH_INFO'])
1180     self.assertEquals('foo=bar', forward_environ['QUERY_STRING'])
1181     self.assertEquals(
1182         ('form-data', {'filename': 'stuff.txt', 'name': 'field1'}),
1183         cgi.parse_header(upload['content-disposition']))
1184
1185   def test_base64(self):
1186     """Test automatic decoding of a base-64-encoded message."""
1187     # Create upload.
1188     upload_data = (
1189         """--================1234==
1190 Content-Type: text/plain
1191 MIME-Version: 1.0
1192 Content-Disposition: form-data; name="field1"; filename="stuff.txt"
1193 Content-Transfer-Encoding: base64
1194
1195 %s
1196 --================1234==--""" % base64.urlsafe_b64encode('value'))
1197
1198     upload_url = blobstore.create_upload_url('/success')
1199
1200     upload, forward_environ, _ = self._run_test_success(
1201         upload_data, upload_url)
1202
1203     self.assertEquals('/success', forward_environ['PATH_INFO'])
1204     self.assertEquals(
1205         ('form-data', {'filename': 'stuff.txt', 'name': 'field1'}),
1206         cgi.parse_header(upload['content-disposition']))
1207
1208   def test_wrong_method(self):
1209     """Using the wrong HTTP method on upload dispatcher causes an error."""
1210     self.environ['REQUEST_METHOD'] = 'GET'
1211
1212     status, _, _, forward_environ, forward_body = self.run_dispatcher()
1213
1214     self.assertEquals('405 Method Not Allowed', status)
1215     # Test that it did not forward.
1216     self.assertEquals(None, forward_environ)
1217     self.assertEquals(None, forward_body)
1218
1219   def test_bad_session(self):
1220     """Using a non-existant upload session causes an error."""
1221     upload_url = blobstore.create_upload_url('/success')
1222
1223     # Get session key from upload url.
1224     session_key = upload_url.split('/')[-1]
1225     datastore.Delete(session_key)
1226
1227     request_path = urlparse.urlparse(upload_url)[2]
1228     self.environ['PATH_INFO'] = request_path
1229     status, _, response_body, forward_environ, forward_body = (
1230         self.run_dispatcher())
1231
1232     self.assertEquals('404 Not Found', status)
1233     self.assertIn('No such upload session: %s' % session_key, response_body)
1234     # Test that it did not forward.
1235     self.assertEquals(None, forward_environ)
1236     self.assertEquals(None, forward_body)
1237
1238   def test_bad_mime_format(self):
1239     """Using a bad mime type format causes an error."""
1240     # Create upload.
1241     upload_data = (
1242         """--================1234==
1243 Content-Type: text/plain/error
1244 MIME-Version: 1.0
1245 Content-Disposition: form-data; name="field1"; filename="stuff.txt"
1246
1247 value
1248 --================1234==--""")
1249
1250     upload_url = blobstore.create_upload_url('/success')
1251
1252     request_path = urlparse.urlparse(upload_url)[2]
1253     self.environ['PATH_INFO'] = request_path
1254     self.environ['CONTENT_TYPE'] = (
1255         'multipart/form-data; boundary="================1234=="')
1256
1257     status, _, response_body, forward_environ, forward_body = (
1258         self.run_dispatcher(upload_data))
1259
1260     self.assertEquals('400 Bad Request', status)
1261     self.assertIn('Incorrectly formatted MIME type: text/plain/error',
1262                   response_body)
1263     # Test that it did not forward.
1264     self.assertEquals(None, forward_environ)
1265     self.assertEquals(None, forward_body)
1266
1267   def test_check_line_endings(self):
1268     """Ensure the upload message uses correct RFC-2821 line terminators."""
1269     # Create upload.
1270     upload_data = (
1271         """--================1234==
1272 Content-Type: text/plain
1273 MIME-Version: 1.0
1274 Content-Disposition: form-data; name="field1"; filename="stuff.txt"
1275
1276 value
1277 --================1234==--""")
1278
1279     upload_url = blobstore.create_upload_url('/success')
1280
1281     request_path = urlparse.urlparse(upload_url)[2]
1282     self.environ['PATH_INFO'] = request_path
1283     self.environ['CONTENT_TYPE'] = (
1284         'multipart/form-data; boundary="================1234=="')
1285
1286     status, _, _, _, forward_body = self.run_dispatcher(upload_data)
1287
1288     self.assertEquals('200 OK', status)
1289     forward_body = forward_body.replace('\r\n', '')
1290     self.assertEqual(forward_body.rfind('\n'), -1)
1291
1292   def test_copy_headers(self):
1293     """Tests that headers are copied, except for ones that should not be."""
1294     # Create upload.
1295     upload_data = (
1296         """--================1234==
1297 Content-Type: text/plain
1298 MIME-Version: 1.0
1299 Content-Disposition: form-data; name="field1"; filename="stuff.txt"
1300
1301 value
1302 --================1234==--""")
1303
1304     upload_url = blobstore.create_upload_url('/success')
1305
1306     request_path = urlparse.urlparse(upload_url)[2]
1307     self.environ['PATH_INFO'] = request_path
1308     self.environ['CONTENT_TYPE'] = (
1309         'multipart/form-data; boundary="================1234=="')
1310     self.environ['HTTP_PLEASE_COPY_ME'] = 'I get copied'
1311     self.environ['HTTP_CONTENT_TYPE'] = 'I should not be copied'
1312     self.environ['HTTP_CONTENT_LENGTH'] = 'I should not be copied'
1313     self.environ['HTTP_CONTENT_MD5'] = 'I should not be copied'
1314
1315     status, _, response_body, forward_environ, forward_body = (
1316         self.run_dispatcher(upload_data))
1317
1318     self.assertEquals('200 OK', status)
1319     self.assertEquals('Forwarded successfully.', response_body)
1320     self.assertIn('HTTP_PLEASE_COPY_ME', forward_environ)
1321     self.assertEquals('I get copied', forward_environ['HTTP_PLEASE_COPY_ME'])
1322     self.assertNotIn('HTTP_CONTENT_TYPE', forward_environ)
1323     self.assertNotIn('HTTP_CONTENT_LENGTH', forward_environ)
1324     self.assertNotIn('HTTP_CONTENT_MD5', forward_environ)
1325     # These ones should have been modified.
1326     self.assertIn('CONTENT_TYPE', forward_environ)
1327     self.assertNotEquals(
1328         'multipart/form-data; boundary="================1234=="',
1329         forward_environ['CONTENT_TYPE'])
1330     self.assertIn('CONTENT_LENGTH', forward_environ)
1331     self.assertEquals(str(len(forward_body)), forward_environ['CONTENT_LENGTH'])
1332
1333   def test_entity_too_large(self):
1334     """Ensure a 413 response is generated when upload size limit exceeded."""
1335     # Create upload.
1336     upload_data = (
1337         """--================1234==
1338 Content-Type: text/plain
1339 MIME-Version: 1.0
1340 Content-Disposition: form-data; name="field1"; filename="stuff.txt"
1341
1342 Lots and Lots of Stuff
1343 --================1234==--""")
1344
1345     upload_url = blobstore.create_upload_url('/success1', max_bytes_per_blob=1)
1346
1347     request_path = urlparse.urlparse(upload_url)[2]
1348     self.environ['PATH_INFO'] = request_path
1349     self.environ['CONTENT_TYPE'] = (
1350         'multipart/form-data; boundary="================1234=="')
1351
1352     status, _, _, forward_environ, forward_body = (
1353         self.run_dispatcher(upload_data))
1354
1355     self.assertEquals('413 Request Entity Too Large', status)
1356     # Test that it did not forward.
1357     self.assertEquals(None, forward_environ)
1358     self.assertEquals(None, forward_body)
1359
1360   def test_filename_too_long(self):
1361     """Ensure a 400 response is generated when filename size limit exceeded."""
1362     filename = 'a' * 500 + '.txt'
1363     # Create upload.
1364     upload_data = (
1365         """Content-Type: multipart/form-data; boundary="================1234=="
1366
1367 --================1234==
1368 Content-Type: text/plain
1369 MIME-Version: 1.0
1370 Content-Disposition: form-data; name="field1"; filename="%s"
1371
1372 Lots and Lots of Stuff
1373 --================1234==--""" % filename)
1374
1375     upload_url = blobstore.create_upload_url('/success1')
1376
1377     request_path = urlparse.urlparse(upload_url)[2]
1378     self.environ['PATH_INFO'] = request_path
1379     self.environ['CONTENT_TYPE'] = (
1380         'multipart/form-data; boundary="================1234=="')
1381
1382     status, _, response_body, forward_environ, forward_body = (
1383         self.run_dispatcher(upload_data))
1384
1385     self.assertEquals('400 Bad Request', status)
1386     self.assertIn('The filename exceeds the maximum allowed length of 500.',
1387                   response_body)
1388     # Test that it did not forward.
1389     self.assertEquals(None, forward_environ)
1390     self.assertEquals(None, forward_body)
1391
1392   def test_content_type_too_long(self):
1393     """Ensure a 400 response when content-type size limit exceeded."""
1394     content_type = 'text/' + 'a' * 500
1395     # Create upload.
1396     upload_data = (
1397         """Content-Type: multipart/form-data; boundary="================1234=="
1398
1399 --================1234==
1400 Content-Type: %s
1401 MIME-Version: 1.0
1402 Content-Disposition: form-data; name="field1"; filename="stuff.txt"
1403
1404 Lots and Lots of Stuff
1405 --================1234==--""" % content_type)
1406
1407     upload_url = blobstore.create_upload_url('/success1')
1408
1409     request_path = urlparse.urlparse(upload_url)[2]
1410     self.environ['PATH_INFO'] = request_path
1411     self.environ['CONTENT_TYPE'] = (
1412         'multipart/form-data; boundary="================1234=="')
1413
1414     status, _, response_body, forward_environ, forward_body = (
1415         self.run_dispatcher(upload_data))
1416
1417     self.assertEquals('400 Bad Request', status)
1418     self.assertIn('The Content-Type exceeds the maximum allowed length of 500.',
1419                   response_body)
1420     # Test that it did not forward.
1421     self.assertEquals(None, forward_environ)
1422     self.assertEquals(None, forward_body)
1423
1424   def test_raise_uncaught_http_error(self):
1425     """Ensure that an uncaught HTTPError is not inadvertently caught."""
1426
1427     def forward_app(unused_environ, unused_start_response):
1428       # Simulate raising a webob.exc.HTTPError in a user's application.
1429       # This should not be caught by our wrapper.
1430       raise webob.exc.HTTPLengthRequired()
1431
1432     self.dispatcher = blob_upload.Application(forward_app)
1433
1434     upload_url = blobstore.create_upload_url('/success')
1435     request_path = urlparse.urlparse(upload_url)[2]
1436     self.environ['PATH_INFO'] = request_path
1437
1438     self.assertRaises(webob.exc.HTTPLengthRequired,
1439                       self.run_dispatcher)
1440
1441
1442 if __name__ == '__main__':
1443   unittest.main()