python/php/sdk/google/appengine/ext/cloud_storage_streams/CloudStorageStreamWrapperTest.php

   1 <?php
   2 /**
   3  * Copyright 2007 Google Inc.
   4  *
   5  * Licensed under the Apache License, Version 2.0 (the "License");
   6  * you may not use this file except in compliance with the License.
   7  * You may obtain a copy of the License at
   8  *
   9  *     http://www.apache.org/licenses/LICENSE-2.0
  10  *
  11  * Unless required by applicable law or agreed to in writing, software
  12  * distributed under the License is distributed on an "AS IS" BASIS,
  13  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  14  * See the License for the specific language governing permissions and
  15  * limitations under the License.
  16  */
  17 /**
  18  * Google Cloud Storage Stream Wrapper Tests.
  19  *
  20  * CodeSniffer does not handle files with multiple namespaces well.
  21  * @codingStandardsIgnoreFile
  22  *
  23  */
  24
  25 namespace {
  26 // Mock Memcache class
  27 class Memcache {
  28   // Mock object to validate calls to memcache
  29   static $mock_memcache = null;
  30
  31   public static function setMockMemcache($mock) {
  32     self::$mock_memcache = $mock;
  33   }
  34   public function get($keys, $flags = null) {
  35     return self::$mock_memcache->get($keys, $flags);
  36   }
  37   public function set($key, $value, $flag = null, $expire = 0) {
  38     return self::$mock_memcache->set($key, $value, $flag, $expire);
  39   }
  40 }
  41
  42 // Mock memcached class, used when invalidating cache entries on write.
  43 class Memcached {
  44   // Mock object to validate calls to memcached
  45   static $mock_memcached = null;
  46
  47   public static function setMockMemcached($mock) {
  48     self::$mock_memcached = $mock;
  49   }
  50
  51   public function deleteMulti($keys, $time = 0) {
  52     self::$mock_memcached->deleteMulti($keys, $time);
  53   }
  54 }
  55
  56 }  // namespace
  57
  58 namespace google\appengine\ext\cloud_storage_streams {
  59
  60 require_once 'google/appengine/api/app_identity/app_identity_service_pb.php';
  61 require_once 'google/appengine/api/app_identity/AppIdentityService.php';
  62 require_once 'google/appengine/api/urlfetch_service_pb.php';
  63 require_once 'google/appengine/ext/cloud_storage_streams/CloudStorageClient.php';
  64 require_once 'google/appengine/ext/cloud_storage_streams/CloudStorageReadClient.php';
  65 require_once 'google/appengine/ext/cloud_storage_streams/CloudStorageStreamWrapper.php';
  66 require_once 'google/appengine/ext/cloud_storage_streams/CloudStorageWriteClient.php';
  67 require_once 'google/appengine/testing/ApiProxyTestBase.php';
  68
  69 use google\appengine\testing\ApiProxyTestBase;
  70 use google\appengine\ext\cloud_storage_streams\CloudStorageClient;
  71 use google\appengine\ext\cloud_storage_streams\CloudStorageReadClient;
  72 use google\appengine\ext\cloud_storage_streams\CloudStorageWriteClient;
  73 use google\appengine\ext\cloud_storage_streams\HttpResponse;
  74 use google\appengine\URLFetchRequest\RequestMethod;
  75
  76 class CloudStorageStreamWrapperTest extends ApiProxyTestBase {
  77
  78   public static $allowed_gs_bucket = "";
  79
  80   protected function setUp() {
  81     parent::setUp();
  82     $this->_SERVER = $_SERVER;
  83
  84     stream_wrapper_register("gs",
  85         "\\google\\appengine\\ext\\cloud_storage_streams\\CloudStorageStreamWrapper",
  86         STREAM_IS_URL);
  87
  88     CloudStorageStreamWrapperTest::$allowed_gs_bucket = "";
  89
  90     // By default disable caching so we don't have to mock out memcache in
  91     // every test
  92     stream_context_set_default(['gs' => ['enable_cache' => false]]);
  93
  94     date_default_timezone_set("UTC");
  95
  96     $this->mock_memcache = $this->getMock('\Memcache');
  97     $this->mock_memcache_call_index = 0;
  98     \Memcache::setMockMemcache($this->mock_memcache);
  99
 100     $this->mock_memcached = $this->getMock('\Memcached');
 101     \Memcached::setMockMemcached($this->mock_memcached);
 102   }
 103
 104   protected function tearDown() {
 105     stream_wrapper_unregister("gs");
 106
 107     $_SERVER = $this->_SERVER;
 108     parent::tearDown();
 109   }
 110
 111   public function testInvalidPathName() {
 112     $this->setExpectedException("\PHPUnit_Framework_Error");
 113     $this->assertFalse(fopen("gs:///object.png", "r"));
 114     $this->setExpectedException("\PHPUnit_Framework_Error");
 115     $this->assertFalse(fopen("gs://", "r"));
 116     $invalid_bucket_names = [
 117         'BadBucketName',
 118         '.another_bad_bucket',
 119         'a',
 120         'goog_bucket',
 121         str_repeat('a', 224),
 122         'a.bucket',
 123         'foobar' . str_repeat('a', 64)
 124     ];
 125     foreach ($invalid_bucket_names as $invalid_name) {
 126       $this->setExpectedException(
 127           "\PHPUnit_Framework_Error",
 128           sprintf("Invalid cloud storage bucket name '%s'", $invalid_name));
 129       $this->assertFalse(fopen(sprintf('gs://%s/file.txt', $invalid_name),
 130                                        'r'));
 131     }
 132   }
 133
 134   public function testInvalidMode() {
 135     $valid_path = "gs://bucket/object_name.png";
 136     $this->setExpectedException("\PHPUnit_Framework_Error");
 137     $this->assertFalse(fopen($valid_path, "r+"));
 138     $this->setExpectedException("\PHPUnit_Framework_Error");
 139     $this->assertFalse(fopen($valid_path, "w+"));
 140     $this->setExpectedException("\PHPUnit_Framework_Error");
 141     $this->assertFalse(fopen($valid_path, "a"));
 142     $this->setExpectedException("\PHPUnit_Framework_Error");
 143     $this->assertFalse(fopen($valid_path, "a+"));
 144     $this->setExpectedException("\PHPUnit_Framework_Error");
 145     $this->assertFalse(fopen($valid_path, "x+"));
 146     $this->setExpectedException("\PHPUnit_Framework_Error");
 147     $this->assertFalse(fopen($valid_path, "c"));
 148     $this->setExpectedException("\PHPUnit_Framework_Error");
 149     $this->assertFalse(fopen($valid_path, "c+"));
 150   }
 151
 152   public function testReadObjectSuccess() {
 153     $body = "Hello from PHP";
 154
 155     $this->expectFileReadRequest($body,
 156                                  0,
 157                                  CloudStorageReadClient::DEFAULT_READ_SIZE,
 158                                  null);
 159
 160     $valid_path = "gs://bucket/object_name.png";
 161     $data = file_get_contents($valid_path);
 162
 163     $this->assertEquals($body, $data);
 164     $this->apiProxyMock->verify();
 165   }
 166
 167   public function testReadObjectCacheHitSuccess() {
 168     $body = "Hello from PHP";
 169
 170     // First call is to create the OAuth token.
 171     $this->expectGetAccessTokenRequest(CloudStorageClient::READ_SCOPE);
 172
 173     // Second call is to retrieve the cached read.
 174     $response = [
 175         'status_code' => 200,
 176         'headers' => [
 177             'Content-Length' => strlen($body),
 178             'ETag' => 'deadbeef',
 179             'Content-Type' => 'text/plain',
 180             'Last-Modified' => 'Mon, 02 Jul 2012 01:41:01 GMT',
 181         ],
 182         'body' => $body,
 183     ];
 184     $this->mock_memcache->expects($this->at($this->mock_memcache_call_index++))
 185                         ->method('get')
 186                         ->with($this->stringStartsWith('_ah_gs_read_cache'))
 187                         ->will($this->returnValue($response));
 188
 189     // We now expect a read request with If-None-Modified set to our etag.
 190     $request_headers = [
 191         'Authorization' => 'OAuth foo token',
 192         'Range' => sprintf('bytes=%d-%d',
 193                            0,
 194                            CloudStorageReadClient::DEFAULT_READ_SIZE - 1),
 195         'If-None-Match' => 'deadbeef',
 196         'x-goog-api-version' => 2,
 197     ];
 198     $response = [
 199         'status_code' => HttpResponse::NOT_MODIFIED,
 200         'headers' => [
 201         ],
 202     ];
 203
 204     $expected_url = $this->makeCloudStorageObjectUrl();
 205     $this->expectHttpRequest($expected_url,
 206                              RequestMethod::GET,
 207                              $request_headers,
 208                              null,
 209                              $response);
 210
 211     $options = [ 'gs' => [
 212             'enable_cache' => true,
 213             'enable_optimistic_cache' => false,
 214         ]
 215     ];
 216     $ctx = stream_context_create($options);
 217     $valid_path = "gs://bucket/object.png";
 218     $data = file_get_contents($valid_path, false, $ctx);
 219
 220     $this->assertEquals($body, $data);
 221     $this->apiProxyMock->verify();
 222   }
 223
 224   public function testReadObjectCacheWriteSuccess() {
 225     $body = "Hello from PHP";
 226
 227     $this->expectFileReadRequest($body,
 228                                  0,
 229                                  CloudStorageReadClient::DEFAULT_READ_SIZE,
 230                                  null);
 231
 232     // Don't read the page from the cache
 233     $this->mock_memcache->expects($this->at($this->mock_memcache_call_index++))
 234                         ->method('get')
 235                         ->with($this->stringStartsWith('_ah_gs_read_cache'))
 236                         ->will($this->returnValue(false));
 237
 238     // Expect a write back to the cache
 239     $cache_expiry_seconds = 60;
 240     $this->mock_memcache->expects($this->at($this->mock_memcache_call_index++))
 241                         ->method('set')
 242                         ->with($this->stringStartsWith('_ah_gs_read_cache'),
 243                                $this->anything(),
 244                                null,
 245                                $cache_expiry_seconds)
 246                         ->will($this->returnValue(false));
 247
 248
 249     $options = [ 'gs' => [
 250             'enable_cache' => true,
 251             'enable_optimistic_cache' => false,
 252             'read_cache_expiry_seconds' => $cache_expiry_seconds,
 253         ]
 254     ];
 255     $ctx = stream_context_create($options);
 256     $valid_path = "gs://bucket/object_name.png";
 257     $data = file_get_contents($valid_path, false, $ctx);
 258
 259     $this->assertEquals($body, $data);
 260     $this->apiProxyMock->verify();
 261   }
 262
 263   public function testReadObjectOptimisiticCacheHitSuccess() {
 264     $body = "Hello from PHP";
 265
 266     // First call is to create the OAuth token.
 267     $this->expectGetAccessTokenRequest(CloudStorageClient::READ_SCOPE);
 268
 269     // Second call is to retrieve the cached read.
 270     $response = [
 271         'status_code' => 200,
 272         'headers' => [
 273             'Content-Length' => strlen($body),
 274             'ETag' => 'deadbeef',
 275             'Content-Type' => 'text/plain',
 276             'Last-Modified' => 'Mon, 02 Jul 2012 01:41:01 GMT',
 277         ],
 278         'body' => $body,
 279     ];
 280     $this->mock_memcache->expects($this->at($this->mock_memcache_call_index++))
 281                         ->method('get')
 282                         ->with($this->stringStartsWith('_ah_gs_read_cache'))
 283                         ->will($this->returnValue($response));
 284
 285     $options = [ 'gs' => [
 286             'enable_cache' => true,
 287             'enable_optimistic_cache' => true,
 288         ]
 289     ];
 290     $ctx = stream_context_create($options);
 291     $valid_path = "gs://bucket/object_name.png";
 292     $data = file_get_contents($valid_path, false, $ctx);
 293
 294     $this->assertEquals($body, $data);
 295     $this->apiProxyMock->verify();
 296   }
 297
 298   public function testReadObjectPartialContentResponseSuccess() {
 299     // GCS returns a 206 even if you can obtain all of the file in the first
 300     // read - this test simulates that behavior.
 301     $body = "Hello from PHP.";
 302
 303     $this->expectFileReadRequest($body,
 304                                  0,
 305                                  CloudStorageReadClient::DEFAULT_READ_SIZE,
 306                                  null,
 307                                  true);
 308
 309     $valid_path = "gs://bucket/object_name.png";
 310     $data = file_get_contents($valid_path);
 311
 312     $this->assertEquals($body, $data);
 313     $this->apiProxyMock->verify();
 314   }
 315
 316   public function testReadLargeObjectSuccess() {
 317     $body = str_repeat("1234567890", 100000);
 318     $data_len = strlen($body);
 319
 320     $read_chunks = ceil($data_len / CloudStorageReadClient::DEFAULT_READ_SIZE);
 321     $start_chunk = 0;
 322     $etag = null;
 323
 324     for ($i = 0; $i < $read_chunks; $i++) {
 325       $this->expectFileReadRequest($body,
 326                                    $start_chunk,
 327                                    CloudStorageReadClient::DEFAULT_READ_SIZE,
 328                                    $etag,
 329                                    true);
 330       $start_chunk += CloudStorageReadClient::DEFAULT_READ_SIZE;
 331       $etag = "deadbeef";
 332     }
 333
 334     $valid_path = "gs://bucket/object_name.png";
 335     $fp = fopen($valid_path, "rt");
 336     $data = stream_get_contents($fp);
 337     fclose($fp);
 338
 339     $this->assertEquals($body, $data);
 340     $this->apiProxyMock->verify();
 341   }
 342
 343   public function testSeekReadObjectSuccess() {
 344     $body = "Hello from PHP";
 345
 346     $this->expectFileReadRequest($body,
 347                                  0,
 348                                  CloudStorageReadClient::DEFAULT_READ_SIZE,
 349                                  null);
 350
 351     $valid_path = "gs://bucket/object_name.png";
 352     $fp = fopen($valid_path, "r");
 353     $this->assertEquals(0, fseek($fp, 4, SEEK_SET));
 354     $this->assertEquals($body[4], fread($fp, 1));
 355     $this->assertEquals(-1, fseek($fp, 100, SEEK_SET));
 356     $this->assertTrue(fclose($fp));
 357
 358     $this->apiProxyMock->verify();
 359   }
 360
 361   public function testReadZeroSizedObjectSuccess() {
 362     $this->expectFileReadRequest("",
 363                                  0,
 364                                  CloudStorageReadClient::DEFAULT_READ_SIZE,
 365                                  null);
 366
 367     $data = file_get_contents("gs://bucket/object_name.png");
 368
 369     $this->assertEquals("", $data);
 370     $this->apiProxyMock->verify();
 371   }
 372
 373   public function testFileSizeSucess() {
 374     $body = "Hello from PHP";
 375
 376     $this->expectFileReadRequest($body,
 377                                  0,
 378                                  CloudStorageReadClient::DEFAULT_READ_SIZE,
 379                                  null);
 380
 381     $valid_path = "gs://bucket/object_name.png";
 382     $fp = fopen($valid_path, "r");
 383     $stat = fstat($fp);
 384     fclose($fp);
 385     $this->assertEquals(strlen($body), $stat["size"]);
 386     $this->apiProxyMock->verify();
 387   }
 388
 389   public function testDeleteObjectSuccess() {
 390     $this->expectGetAccessTokenRequest(CloudStorageClient::WRITE_SCOPE);
 391
 392     $request_headers = $this->getStandardRequestHeaders();
 393     $response = [
 394         'status_code' => 204,
 395         'headers' => [
 396         ],
 397     ];
 398     $expected_url = $this->makeCloudStorageObjectUrl("my_bucket",
 399                                                      "/some%file.txt");
 400     $this->expectHttpRequest($expected_url,
 401                              RequestMethod::DELETE,
 402                              $request_headers,
 403                              null,
 404                              $response);
 405
 406     $this->assertTrue(unlink("gs://my_bucket/some%file.txt"));
 407     $this->apiProxyMock->verify();
 408   }
 409
 410   public function testDeleteObjectFail() {
 411     $this->expectGetAccessTokenRequest(CloudStorageClient::WRITE_SCOPE);
 412
 413     $request_headers = $this->getStandardRequestHeaders();
 414     $response = [
 415         'status_code' => 404,
 416         'headers' => [
 417         ],
 418         'body' => "<?xml version='1.0' encoding='utf-8'?>
 419                    <Error>
 420                    <Code>NoSuchBucket</Code>
 421                    <Message>No Such Bucket</Message>
 422                    </Error>",
 423     ];
 424     $expected_url = $this->makeCloudStorageObjectUrl();
 425     $this->expectHttpRequest($expected_url,
 426                              RequestMethod::DELETE,
 427                              $request_headers,
 428                              null,
 429                              $response);
 430
 431     $this->setExpectedException("PHPUnit_Framework_Error_Warning",
 432         "Cloud Storage Error: No Such Bucket (NoSuchBucket)");
 433     $this->assertFalse(unlink("gs://bucket/object.png"));
 434     $this->apiProxyMock->verify();
 435   }
 436
 437   public function testStatBucketSuccess() {
 438     $this->expectGetAccessTokenRequest(CloudStorageClient::READ_SCOPE);
 439     $request_headers = $this->getStandardRequestHeaders();
 440     $file_results = ['file1.txt', 'file2.txt'];
 441     $response = [
 442         'status_code' => 200,
 443         'headers' => [
 444         ],
 445         'body' => $this->makeGetBucketXmlResponse("", $file_results),
 446     ];
 447     $expected_url = $this->makeCloudStorageObjectUrl("bucket", null);
 448     $expected_query = http_build_query([
 449         "delimiter" => CloudStorageClient::DELIMITER,
 450         "max-keys" => CloudStorageUrlStatClient::MAX_KEYS,
 451     ]);
 452
 453     $this->expectHttpRequest(sprintf("%s?%s", $expected_url, $expected_query),
 454                              RequestMethod::GET,
 455                              $request_headers,
 456                              null,
 457                              $response);
 458
 459     // Return a false is writable check from the cache
 460     $this->expectIsWritableMemcacheLookup(true, false);
 461
 462     $this->assertTrue(is_dir("gs://bucket"));
 463     $this->apiProxyMock->verify();
 464   }
 465
 466   public function testStatObjectSuccess() {
 467     $this->expectGetAccessTokenRequest(CloudStorageClient::READ_SCOPE);
 468     // Return the object we want in the second request so we test fetching from
 469     // the marker to get all of the results
 470     $last_modified = 'Mon, 01 Jul 2013 10:02:46 GMT';
 471     $request_headers = $this->getStandardRequestHeaders();
 472     $file_results = [
 473         ['key' => 'object1.png', 'size' => '3337', 'mtime' => $last_modified],
 474     ];
 475     $response = [
 476         'status_code' => 200,
 477         'headers' => [
 478         ],
 479         'body' => $this->makeGetBucketXmlResponse("", $file_results, "foo"),
 480     ];
 481     $expected_url = $this->makeCloudStorageObjectUrl("bucket", null);
 482     $expected_query = http_build_query([
 483         'delimiter' => CloudStorageClient::DELIMITER,
 484         'max-keys' => CloudStorageUrlStatClient::MAX_KEYS,
 485         'prefix' => 'object.png',
 486     ]);
 487
 488     $this->expectHttpRequest(sprintf("%s?%s", $expected_url, $expected_query),
 489                              RequestMethod::GET,
 490                              $request_headers,
 491                              null,
 492                              $response);
 493
 494     $this->expectGetAccessTokenRequest(CloudStorageClient::READ_SCOPE);
 495     $file_results = [
 496         ['key' => 'object.png', 'size' => '37337', 'mtime' => $last_modified],
 497     ];
 498     $response['body'] = $this->makeGetBucketXmlResponse("", $file_results);
 499     $expected_query = http_build_query([
 500         'delimiter' => CloudStorageClient::DELIMITER,
 501         'max-keys' => CloudStorageUrlStatClient::MAX_KEYS,
 502         'prefix' => 'object.png',
 503         'marker' => 'foo',
 504     ]);
 505     $this->expectHttpRequest(sprintf("%s?%s", $expected_url, $expected_query),
 506                              RequestMethod::GET,
 507                              $request_headers,
 508                              null,
 509                              $response);
 510
 511     // Don't find the key in the cache, to force a write attempt to the bucket.
 512     $temp_url = $this->makeCloudStorageObjectUrl("bucket",
 513         CloudStorageClient::WRITABLE_TEMP_FILENAME);
 514     $this->expectIsWritableMemcacheLookup(false, false);
 515     $this->expectFileWriteStartRequest(null, null, 'foo', $temp_url, null);
 516     $this->expectIsWritableMemcacheSet(true);
 517
 518
 519     $result = stat("gs://bucket/object.png");
 520     $this->assertEquals(37337, $result['size']);
 521     $this->assertEquals(0100666, $result['mode']);
 522     $this->assertEquals(strtotime($last_modified), $result['mtime']);
 523     $this->apiProxyMock->verify();
 524   }
 525
 526   public function testStatObjectAsFolderSuccess() {
 527     $this->expectGetAccessTokenRequest(CloudStorageClient::READ_SCOPE);
 528     $request_headers = $this->getStandardRequestHeaders();
 529     $last_modified = 'Mon, 01 Jul 2013 10:02:46 GMT';
 530     $file_results = [
 531         ['key' => 'a/b_$folder$', 'size' => '0', 'mtime' => $last_modified],
 532     ];
 533     $response = [
 534         'status_code' => 200,
 535         'headers' => [
 536         ],
 537         'body' => $this->makeGetBucketXmlResponse('a/b', $file_results),
 538     ];
 539     $expected_url = $this->makeCloudStorageObjectUrl('bucket', null);
 540     $expected_query = http_build_query([
 541         'delimiter' => CloudStorageClient::DELIMITER,
 542         'max-keys' => CloudStorageUrlStatClient::MAX_KEYS,
 543         'prefix' => 'a/b',
 544     ]);
 545
 546     $this->expectHttpRequest(sprintf("%s?%s", $expected_url, $expected_query),
 547                              RequestMethod::GET,
 548                              $request_headers,
 549                              null,
 550                              $response);
 551     // Return a false is writable check from the cache
 552     $this->expectIsWritableMemcacheLookup(true, false);
 553
 554     $this->assertTrue(is_dir('gs://bucket/a/b/'));
 555     $this->apiProxyMock->verify();
 556   }
 557
 558   public function testStatObjectWithCommonPrefixSuccess() {
 559     $this->expectGetAccessTokenRequest(CloudStorageClient::READ_SCOPE);
 560     $request_headers = $this->getStandardRequestHeaders();
 561     $last_modified = 'Mon, 01 Jul 2013 10:02:46 GMT';
 562     $common_prefix_results = ['a/b/c/',
 563         'a/b/d/',
 564     ];
 565     $response = [
 566         'status_code' => 200,
 567         'headers' => [
 568         ],
 569         'body' => $this->makeGetBucketXmlResponse('a/b',
 570                                                   [],
 571                                                   null,
 572                                                   $common_prefix_results),
 573     ];
 574     $expected_url = $this->makeCloudStorageObjectUrl('bucket', null);
 575     $expected_query = http_build_query([
 576         'delimiter' => CloudStorageClient::DELIMITER,
 577         'max-keys' => CloudStorageUrlStatClient::MAX_KEYS,
 578         'prefix' => 'a/b',
 579     ]);
 580
 581     $this->expectHttpRequest(sprintf("%s?%s", $expected_url, $expected_query),
 582                              RequestMethod::GET,
 583                              $request_headers,
 584                              null,
 585                              $response);
 586     // Return a false is writable check from the cache
 587     $this->expectIsWritableMemcacheLookup(true, false);
 588
 589     $this->assertTrue(is_dir('gs://bucket/a/b'));
 590     $this->apiProxyMock->verify();
 591   }
 592
 593   public function testStatObjectFailed() {
 594     $this->expectGetAccessTokenRequest(CloudStorageClient::READ_SCOPE);
 595     $request_headers = $this->getStandardRequestHeaders();
 596     $response = [
 597         'status_code' => 404,
 598         'headers' => [
 599         ],
 600     ];
 601     $expected_url = $this->makeCloudStorageObjectUrl("bucket", null);
 602     $expected_query = http_build_query([
 603         'delimiter' => CloudStorageClient::DELIMITER,
 604         'max-keys' => CloudStorageUrlStatClient::MAX_KEYS,
 605         'prefix' => 'object.png',
 606     ]);
 607
 608     $this->expectHttpRequest(sprintf("%s?%s", $expected_url, $expected_query),
 609                              RequestMethod::GET,
 610                              $request_headers,
 611                              null,
 612                              $response);
 613
 614     $this->setExpectedException("PHPUnit_Framework_Error_Warning");
 615     $result = stat("gs://bucket/object.png");
 616     $this->apiProxyMock->verify();
 617   }
 618
 619   public function testInvalidRenamePaths() {
 620     $this->setExpectedException("\PHPUnit_Framework_Error");
 621     $this->assertFalse(rename("gs://bucket/object.png", "gs://to/"));
 622     $this->setExpectedException("\PHPUnit_Framework_Error");
 623     $this->assertFalse(rename("gs://bucket/", "gs://to/object.png"));
 624   }
 625
 626   public function testRenameObjectSuccess() {
 627     $this->expectGetAccessTokenRequest(CloudStorageClient::WRITE_SCOPE);
 628
 629     // First there is a stat
 630     $request_headers = $this->getStandardRequestHeaders();
 631     $response = [
 632         'status_code' => 200,
 633         'headers' => [
 634             'Content-Length' => 37337,
 635             'ETag' => 'abcdef',
 636             'Content-Type' => 'text/plain',
 637         ],
 638     ];
 639
 640     $expected_url = $this->makeCloudStorageObjectUrl();
 641     $this->expectHttpRequest($expected_url,
 642                              RequestMethod::HEAD,
 643                              $request_headers,
 644                              null,
 645                              $response);
 646
 647     // Then there is a copy
 648     $request_headers = [
 649         "Authorization" => "OAuth foo token",
 650         "x-goog-copy-source" => '/bucket/object.png',
 651         "x-goog-copy-source-if-match" => 'abcdef',
 652         "content-type" => 'text/plain',
 653         "x-goog-metadata-directive" => "COPY",
 654         "x-goog-api-version" => 2,
 655     ];
 656     $response = [
 657         'status_code' => 200,
 658         'headers' => [
 659         ]
 660     ];
 661     $expected_url = $this->makeCloudStorageObjectUrl("to_bucket", "/to.png");
 662     $this->expectHttpRequest($expected_url,
 663                              RequestMethod::PUT,
 664                              $request_headers,
 665                              null,
 666                              $response);
 667
 668     // Then we unlink the original.
 669     $request_headers = $this->getStandardRequestHeaders();
 670     $response = [
 671         'status_code' => 204,
 672         'headers' => [
 673         ],
 674     ];
 675     $expected_url = $this->makeCloudStorageObjectUrl();
 676     $this->expectHttpRequest($expected_url,
 677                              RequestMethod::DELETE,
 678                              $request_headers,
 679                              null,
 680                              $response);
 681
 682     $from = "gs://bucket/object.png";
 683     $to = "gs://to_bucket/to.png";
 684
 685     $this->assertTrue(rename($from, $to));
 686     $this->apiProxyMock->verify();
 687   }
 688
 689   public function testRenameObjectFromObjectNotFound() {
 690     $this->expectGetAccessTokenRequest(CloudStorageClient::WRITE_SCOPE);
 691
 692     // First there is a stat
 693     $request_headers = $this->getStandardRequestHeaders();
 694     $response = [
 695         'status_code' => 404,
 696         'headers' => [
 697         ],
 698     ];
 699
 700     $expected_url = $this->makeCloudStorageObjectUrl();
 701     $this->expectHttpRequest($expected_url,
 702                              RequestMethod::HEAD,
 703                              $request_headers,
 704                              null,
 705                              $response);
 706
 707     $from = "gs://bucket/object.png";
 708     $to = "gs://to_bucket/to_object";
 709
 710     $this->setExpectedException("\PHPUnit_Framework_Error_Warning");
 711     rename($from, $to);
 712     $this->apiProxyMock->verify();
 713   }
 714
 715   public function testRenameObjectCopyFailed() {
 716     $this->expectGetAccessTokenRequest(CloudStorageClient::WRITE_SCOPE);
 717
 718     // First there is a stat
 719     $request_headers = $this->getStandardRequestHeaders();
 720     $response = [
 721         'status_code' => 200,
 722         'headers' => [
 723             'Content-Length' => 37337,
 724             'ETag' => 'abcdef',
 725             'Content-Type' => 'text/plain',
 726         ],
 727     ];
 728
 729     $expected_url = $this->makeCloudStorageObjectUrl();
 730     $this->expectHttpRequest($expected_url,
 731                              RequestMethod::HEAD,
 732                              $request_headers,
 733                              null,
 734                              $response);
 735
 736     // Then there is a copy
 737     $request_headers = [
 738         "Authorization" => "OAuth foo token",
 739         "x-goog-copy-source" => '/bucket/object.png',
 740         "x-goog-copy-source-if-match" => 'abcdef',
 741         "content-type" => 'text/plain',
 742         "x-goog-metadata-directive" => "COPY",
 743         "x-goog-api-version" => 2,
 744     ];
 745     $response = [
 746         'status_code' => 412,
 747         'headers' => [
 748         ]
 749     ];
 750     $expected_url = $this->makeCloudStorageObjectUrl("to_bucket", "/to_object");
 751     $this->expectHttpRequest($expected_url,
 752                              RequestMethod::PUT,
 753                              $request_headers,
 754                              null,
 755                              $response);
 756
 757     $from = "gs://bucket/object.png";
 758     $to = "gs://to_bucket/to_object";
 759
 760     $this->setExpectedException("\PHPUnit_Framework_Error_Warning");
 761     $this->assertFalse(rename($from, $to));
 762     $this->apiProxyMock->verify();
 763   }
 764
 765   public function testRenameObjectUnlinkFailed() {
 766     $this->expectGetAccessTokenRequest(CloudStorageClient::WRITE_SCOPE);
 767
 768     // First there is a stat
 769     $request_headers = $this->getStandardRequestHeaders();
 770     $response = [
 771         'status_code' => 200,
 772         'headers' => [
 773             'Content-Length' => 37337,
 774             'ETag' => 'abcdef',
 775             'Content-Type' => 'text/plain',
 776         ],
 777     ];
 778
 779     $expected_url = $this->makeCloudStorageObjectUrl();
 780     $this->expectHttpRequest($expected_url,
 781                              RequestMethod::HEAD,
 782                              $request_headers,
 783                              null,
 784                              $response);
 785
 786     // Then there is a copy
 787     $request_headers = [
 788         "Authorization" => "OAuth foo token",
 789         "x-goog-copy-source" => '/bucket/object.png',
 790         "x-goog-copy-source-if-match" => 'abcdef',
 791         "content-type" => 'text/plain',
 792         "x-goog-metadata-directive" => "COPY",
 793         "x-goog-api-version" => 2,
 794     ];
 795     $response = [
 796         'status_code' => 200,
 797         'headers' => [
 798         ]
 799     ];
 800     $expected_url = $this->makeCloudStorageObjectUrl("to_bucket",
 801                                                      "/to_object");
 802     $this->expectHttpRequest($expected_url,
 803                              RequestMethod::PUT,
 804                              $request_headers,
 805                              null,
 806                              $response);
 807
 808     // Then we unlink the original.
 809     $request_headers = $this->getStandardRequestHeaders();
 810     $response = [
 811         'status_code' => 404,
 812         'headers' => [
 813         ],
 814     ];
 815     $expected_url = $this->makeCloudStorageObjectUrl();
 816     $this->expectHttpRequest($expected_url,
 817                              RequestMethod::DELETE,
 818                              $request_headers,
 819                              null,
 820                              $response);
 821
 822     $from = "gs://bucket/object.png";
 823     $to = "gs://to_bucket/to_object";
 824
 825     $this->setExpectedException("\PHPUnit_Framework_Error_Warning");
 826     $this->assertFalse(rename($from, $to));
 827     $this->apiProxyMock->verify();
 828   }
 829
 830   public function testWriteObjectSuccess() {
 831     $this->writeObjectSuccessWithMetadata("Hello To PHP.");
 832   }
 833
 834   public function testWriteObjectWithMetadata() {
 835     $metadata = ["foo" => "far", "bar" => "boo"];
 836     $this->writeObjectSuccessWithMetadata("Goodbye To PHP.", $metadata);
 837   }
 838
 839   private function writeObjectSuccessWithMetadata($data, $metadata = NULL) {
 840     $data_len = strlen($data);
 841     $expected_url = $this->makeCloudStorageObjectUrl();
 842     $this->expectFileWriteStartRequest("text/plain",
 843                                        "public-read",
 844                                        "foo_upload_id",
 845                                        $expected_url,
 846                                        $metadata);
 847
 848     $this->expectFileWriteContentRequest($expected_url,
 849                                          "foo_upload_id",
 850                                          $data,
 851                                          0,
 852                                          $data_len - 1,
 853                                          true);
 854     $context = [
 855         "gs" => [
 856             "acl" => "public-read",
 857             "Content-Type" => "text/plain",
 858             'enable_cache' => true,
 859         ],
 860     ];
 861     if (isset($metadata)) {
 862       $context["gs"]["metadata"] = $metadata;
 863     }
 864
 865     $range = sprintf("bytes=0-%d", CloudStorageClient::DEFAULT_READ_SIZE - 1);
 866     $cache_key = sprintf(CloudStorageClient::MEMCACHE_KEY_FORMAT,
 867                          $expected_url,
 868                          $range);
 869     $this->mock_memcached->expects($this->once())
 870                          ->method('deleteMulti')
 871                          ->with($this->identicalTo([$cache_key]));
 872
 873     stream_context_set_default($context);
 874     $this->assertEquals($data_len,
 875         file_put_contents("gs://bucket/object.png", $data));
 876     $this->apiProxyMock->verify();
 877   }
 878
 879   public function testWriteInvalidMetadata() {
 880     $metadata = ["f o o" => "far"];
 881     $context = [
 882         "gs" => [
 883             "acl" => "public-read",
 884             "Content-Type" => "text/plain",
 885             "metadata" => $metadata
 886         ],
 887     ];
 888     stream_context_set_default($context);
 889     $this->expectGetAccessTokenRequest(CloudStorageClient::WRITE_SCOPE);
 890     $this->setExpectedException("\PHPUnit_Framework_Error");
 891     file_put_contents("gs://bucket/object.png", "Some data");
 892   }
 893
 894   public function testWriteLargeObjectSuccess() {
 895     $data_to_write = str_repeat("1234567890", 100000);
 896     $data_len = strlen($data_to_write);
 897
 898     $expected_url = $this->makeCloudStorageObjectUrl();
 899
 900     $this->expectFileWriteStartRequest("text/plain",
 901                                        "public-read",
 902                                        "foo_upload_id",
 903                                        $expected_url);
 904
 905     $chunks = floor($data_len / CloudStorageWriteClient::WRITE_CHUNK_SIZE);
 906     $start_byte = 0;
 907     $end_byte = CloudStorageWriteClient::WRITE_CHUNK_SIZE - 1;
 908
 909     for ($i = 0 ; $i < $chunks ; $i++) {
 910       $this->expectFileWriteContentRequest($expected_url,
 911                                            "foo_upload_id",
 912                                            $data_to_write,
 913                                            $start_byte,
 914                                            $end_byte,
 915                                            false);
 916       $start_byte += CloudStorageWriteClient::WRITE_CHUNK_SIZE;
 917       $end_byte += CloudStorageWriteClient::WRITE_CHUNK_SIZE;
 918     }
 919
 920     // Write out the remainder
 921     $this->expectFileWriteContentRequest($expected_url,
 922                                          "foo_upload_id",
 923                                          $data_to_write,
 924                                          $start_byte,
 925                                          $data_len - 1,
 926                                          true);
 927
 928     $file_context = [
 929         "gs" => [
 930             "acl" => "public-read",
 931             "Content-Type" => "text/plain",
 932             'enable_cache' => true,
 933         ],
 934     ];
 935
 936     $delete_keys = [];
 937     for ($i = 0; $i < $data_len; $i += CloudStorageClient::DEFAULT_READ_SIZE) {
 938       $range = sprintf("bytes=%d-%d",
 939                        $i,
 940                        $i + CloudStorageClient::DEFAULT_READ_SIZE - 1);
 941       $delete_keys[] = sprintf(CloudStorageClient::MEMCACHE_KEY_FORMAT,
 942                                $expected_url,
 943                                $range);
 944     }
 945     $this->mock_memcached->expects($this->once())
 946                          ->method('deleteMulti')
 947                          ->with($this->identicalTo($delete_keys));
 948
 949     $ctx = stream_context_create($file_context);
 950     $this->assertEquals($data_len,
 951                         file_put_contents("gs://bucket/object.png",
 952                                           $data_to_write,
 953                                           0,
 954                                           $ctx));
 955     $this->apiProxyMock->verify();
 956   }
 957
 958   public function testWriteEmptyObjectSuccess() {
 959     $data_to_write = "";
 960     $data_len = 0;
 961
 962     $expected_url = $this->makeCloudStorageObjectUrl("bucket",
 963                                                      "/empty_file.txt");
 964
 965     $this->expectFileWriteStartRequest("text/plain",
 966                                        "public-read",
 967                                        "foo_upload_id",
 968                                        $expected_url);
 969
 970     $this->expectFileWriteContentRequest($expected_url,
 971                                          "foo_upload_id",
 972                                          $data_to_write,
 973                                          null,  // start_byte
 974                                          0,  // write_length
 975                                          true);  // Complete write
 976
 977     $file_context = [
 978         "gs" => [
 979             "acl" => "public-read",
 980             "Content-Type" => "text/plain",
 981         ],
 982     ];
 983     $ctx = stream_context_create($file_context);
 984     $fp = fopen("gs://bucket/empty_file.txt", "wt", false, $ctx);
 985     $this->assertEquals($data_len, fwrite($fp, $data_to_write));
 986     fclose($fp);
 987     $this->apiProxyMock->verify();
 988   }
 989
 990   public function testInvalidBucketForInclude() {
 991     stream_wrapper_unregister("gs");
 992     stream_wrapper_register("gs",
 993         "\\google\\appengine\\ext\\cloud_storage_streams\\CloudStorageStreamWrapper",
 994         0);
 995
 996     $this->setExpectedException("\PHPUnit_Framework_Error");
 997     include 'gs://bucket/object.php';
 998   }
 999
1000   public function testValidBucketForInclude() {
1001     stream_wrapper_unregister("gs");
1002     stream_wrapper_register("gs",
1003         "\\google\\appengine\\ext\\cloud_storage_streams\\CloudStorageStreamWrapper",
1004         0);
1005
1006     $body = '<?php $a = "foo";';
1007
1008     $this->expectFileReadRequest($body,
1009                                  0,
1010                                  CloudStorageReadClient::DEFAULT_READ_SIZE,
1011                                  null);
1012
1013     define("GAE_INCLUDE_GS_BUCKETS", "foo, bucket, bar");
1014     $valid_path = "gs://bucket/object_name.png";
1015     require $valid_path;
1016
1017     $this->assertEquals($a, 'foo');
1018     $this->apiProxyMock->verify();
1019   }
1020
1021   public function testInvalidPathForOpenDir() {
1022     $this->setExpectedException("\PHPUnit_Framework_Error");
1023     $this->assertFalse(opendir("gs:///"));
1024     $this->setExpectedException("\PHPUnit_Framework_Error");
1025     $this->assertFalse(fopen("gs://"));
1026   }
1027
1028   public function testReaddirSuccess() {
1029     $this->expectGetAccessTokenRequest(CloudStorageClient::READ_SCOPE);
1030
1031     $request_headers = $this->getStandardRequestHeaders();
1032     $file_results = ['f/file1.txt', 'f/file2.txt', 'f/sub_$folder$'];
1033     $response = [
1034         'status_code' => 200,
1035         'headers' => [
1036         ],
1037         'body' => $this->makeGetBucketXmlResponse("f/", $file_results),
1038     ];
1039     $expected_url = $this->makeCloudStorageObjectUrl("bucket", null);
1040     $expected_query = http_build_query([
1041         "delimiter" => CloudStorageDirectoryClient::DELIMITER,
1042         "max-keys" => CloudStorageDirectoryClient::MAX_KEYS,
1043         "prefix" => "f/",
1044     ]);
1045
1046     $this->expectHttpRequest(sprintf("%s?%s", $expected_url, $expected_query),
1047                              RequestMethod::GET,
1048                              $request_headers,
1049                              null,
1050                              $response);
1051
1052     $res = opendir("gs://bucket/f");
1053     $this->assertEquals("file1.txt", readdir($res));
1054     $this->assertEquals("file2.txt", readdir($res));
1055     $this->assertEquals("sub/", readdir($res));
1056     $this->assertFalse(readdir($res));
1057     closedir($res);
1058     $this->apiProxyMock->verify();
1059   }
1060
1061   public function testReaddirTruncatedSuccess() {
1062     $this->expectGetAccessTokenRequest(CloudStorageClient::READ_SCOPE);
1063     $request_headers = $this->getStandardRequestHeaders();
1064     // First query with a truncated response
1065     $response_body = "<?xml version='1.0' encoding='UTF-8'?>
1066         <ListBucketResult xmlns='http://doc.s3.amazonaws.com/2006-03-01'>
1067         <Name>sjl-test</Name>
1068         <Prefix>f/</Prefix>
1069         <Marker></Marker>
1070         <NextMarker>AA</NextMarker>
1071         <Delimiter>/</Delimiter>
1072         <IsTruncated>true</IsTruncated>
1073         <Contents>
1074           <Key>f/file1.txt</Key>
1075         </Contents>
1076         <Contents>
1077           <Key>f/file2.txt</Key>
1078         </Contents>
1079         </ListBucketResult>";
1080     $response = [
1081         'status_code' => 200,
1082         'headers' => [
1083         ],
1084         'body' => $response_body,
1085     ];
1086     $expected_url = $this->makeCloudStorageObjectUrl("bucket", null);
1087     $expected_query = http_build_query([
1088         "delimiter" => CloudStorageDirectoryClient::DELIMITER,
1089         "max-keys" => CloudStorageDirectoryClient::MAX_KEYS,
1090         "prefix" => "f/",
1091     ]);
1092
1093     $this->expectHttpRequest(sprintf("%s?%s", $expected_url, $expected_query),
1094                              RequestMethod::GET,
1095                              $request_headers,
1096                              null,
1097                              $response);
1098
1099     // Second query with the remaining response
1100     $response_body = "<?xml version='1.0' encoding='UTF-8'?>
1101         <ListBucketResult xmlns='http://doc.s3.amazonaws.com/2006-03-01'>
1102         <Name>sjl-test</Name>
1103         <Prefix>f/</Prefix>
1104         <Marker>AA</Marker>
1105         <Delimiter>/</Delimiter>
1106         <IsTruncated>false</IsTruncated>
1107         <Contents>
1108           <Key>f/file3.txt</Key>
1109         </Contents>
1110         <Contents>
1111           <Key>f/file4.txt</Key>
1112         </Contents>
1113         </ListBucketResult>";
1114     $response = [
1115         'status_code' => 200,
1116         'headers' => [
1117         ],
1118         'body' => $response_body,
1119     ];
1120
1121     $expected_query = http_build_query([
1122         "delimiter" => CloudStorageDirectoryClient::DELIMITER,
1123         "max-keys" => CloudStorageDirectoryClient::MAX_KEYS,
1124         "prefix" => "f/",
1125         "marker" => "AA",
1126     ]);
1127
1128     $this->expectGetAccessTokenRequest(CloudStorageClient::READ_SCOPE);
1129     $this->expectHttpRequest(sprintf("%s?%s", $expected_url, $expected_query),
1130                              RequestMethod::GET,
1131                              $request_headers,
1132                              null,
1133                              $response);
1134
1135     $res = opendir("gs://bucket/f");
1136     $this->assertEquals("file1.txt", readdir($res));
1137     $this->assertEquals("file2.txt", readdir($res));
1138     $this->assertEquals("file3.txt", readdir($res));
1139     $this->assertEquals("file4.txt", readdir($res));
1140     $this->assertFalse(readdir($res));
1141     closedir($res);
1142     $this->apiProxyMock->verify();
1143   }
1144
1145   public function testRewindDirSuccess() {
1146     $this->expectGetAccessTokenRequest(CloudStorageClient::READ_SCOPE);
1147     $request_headers = $this->getStandardRequestHeaders();
1148     $response = [
1149         'status_code' => 200,
1150         'headers' => [
1151         ],
1152         'body' => $this->makeGetBucketXmlResponse(
1153             "f/",
1154             ["f/file1.txt", "f/file2.txt"]),
1155     ];
1156     $expected_url = $this->makeCloudStorageObjectUrl("bucket", null);
1157     $expected_query = http_build_query([
1158         "delimiter" => CloudStorageDirectoryClient::DELIMITER,
1159         "max-keys" => CloudStorageDirectoryClient::MAX_KEYS,
1160         "prefix" => "f/",
1161     ]);
1162
1163     $this->expectHttpRequest(sprintf("%s?%s", $expected_url, $expected_query),
1164                              RequestMethod::GET,
1165                              $request_headers,
1166                              null,
1167                              $response);
1168     // Expect the requests again when we rewinddir
1169     $this->expectGetAccessTokenRequest(CloudStorageClient::READ_SCOPE);
1170     $this->expectHttpRequest(sprintf("%s?%s", $expected_url, $expected_query),
1171                              RequestMethod::GET,
1172                              $request_headers,
1173                              null,
1174                              $response);
1175
1176     $res = opendir("gs://bucket/f");
1177     $this->assertEquals("file1.txt", readdir($res));
1178     rewinddir($res);
1179     $this->assertEquals("file1.txt", readdir($res));
1180     $this->assertEquals("file2.txt", readdir($res));
1181     $this->assertFalse(readdir($res));
1182     closedir($res);
1183     $this->apiProxyMock->verify();
1184   }
1185
1186   public function testInvalidPathForMkDir() {
1187     $this->setExpectedException("\PHPUnit_Framework_Error");
1188     $this->assertFalse(mkdir("gs://bucket_without_object/"));
1189     $this->setExpectedException("\PHPUnit_Framework_Error");
1190     $this->assertFalse(mkdir("gs://"));
1191   }
1192
1193   public function testMkDirSuccess() {
1194     $this->expectGetAccessTokenRequest(CloudStorageClient::WRITE_SCOPE);
1195     $request_headers = [
1196         "Authorization" => "OAuth foo token",
1197         "x-goog-if-generation-match" => 0,
1198         "Content-Range" => "bytes */0",
1199         "x-goog-api-version" => 2,
1200     ];
1201
1202     $response = [
1203         'status_code' => 200,
1204         'headers' => [
1205         ],
1206     ];
1207
1208     $expected_url = $this->makeCloudStorageObjectUrl('bucket',
1209                                                      '/dira/dirb_$folder$');
1210     $this->expectHttpRequest($expected_url,
1211                              RequestMethod::PUT,
1212                              $request_headers,
1213                              null,
1214                              $response);
1215
1216     $this->assertTrue(mkdir("gs://bucket/dira/dirb"));
1217     $this->apiProxyMock->verify();
1218   }
1219
1220   public function testInvalidPathForRmDir() {
1221     $this->setExpectedException("\PHPUnit_Framework_Error");
1222     $this->assertFalse(rmdir("gs://bucket_without_object/"));
1223     $this->setExpectedException("\PHPUnit_Framework_Error");
1224     $this->assertFalse(rmdir("gs://"));
1225   }
1226
1227   public function testRmDirSuccess() {
1228     // Expect a request to list the contents of the bucket to ensure that it is
1229     // empty.
1230     $this->expectGetAccessTokenRequest(CloudStorageClient::READ_SCOPE);
1231     $request_headers = $this->getStandardRequestHeaders();
1232     // First query with a truncated response
1233     $response = [
1234         'status_code' => 200,
1235         'headers' => [
1236         ],
1237         'body' => $this->makeGetBucketXmlResponse("dira/dirb/", []),
1238     ];
1239     $expected_url = $this->makeCloudStorageObjectUrl("bucket", null);
1240     $expected_query = http_build_query([
1241         "delimiter" => CloudStorageDirectoryClient::DELIMITER,
1242         "max-keys" => CloudStorageDirectoryClient::MAX_KEYS,
1243         "prefix" => "dira/dirb/",
1244     ]);
1245
1246     $this->expectHttpRequest(sprintf("%s?%s", $expected_url, $expected_query),
1247                              RequestMethod::GET,
1248                              $request_headers,
1249                              null,
1250                              $response);
1251
1252     // Expect the unlink request for the folder.
1253     $this->expectGetAccessTokenRequest(CloudStorageClient::WRITE_SCOPE);
1254     $request_headers = $this->getStandardRequestHeaders();
1255     $response = [
1256         'status_code' => 204,
1257         'headers' => [
1258         ],
1259     ];
1260
1261     $expected_url = $this->makeCloudStorageObjectUrl('bucket',
1262                                                      '/dira/dirb_$folder$');
1263     $this->expectHttpRequest($expected_url,
1264                              RequestMethod::DELETE,
1265                              $request_headers,
1266                              null,
1267                              $response);
1268
1269     $this->assertTrue(rmdir("gs://bucket/dira/dirb"));
1270     $this->apiProxyMock->verify();
1271   }
1272
1273   public function testRmDirNotEmpry() {
1274     // Expect a request to list the contents of the bucket to ensure that it is
1275     // empty.
1276     $this->expectGetAccessTokenRequest(CloudStorageClient::READ_SCOPE);
1277     $request_headers = $this->getStandardRequestHeaders();
1278     // First query with a truncated response
1279     $response = [
1280         'status_code' => 200,
1281         'headers' => [
1282         ],
1283         'body' => $this->makeGetBucketXmlResponse(
1284             "dira/dirb/",
1285             ["dira/dirb/file1.txt"]),
1286     ];
1287     $expected_url = $this->makeCloudStorageObjectUrl("bucket", null);
1288     $expected_query = http_build_query([
1289         "delimiter" => CloudStorageDirectoryClient::DELIMITER,
1290         "max-keys" => CloudStorageDirectoryClient::MAX_KEYS,
1291         "prefix" => "dira/dirb/",
1292     ]);
1293
1294     $this->expectHttpRequest(sprintf("%s?%s", $expected_url, $expected_query),
1295                              RequestMethod::GET,
1296                              $request_headers,
1297                              null,
1298                              $response);
1299
1300     $this->setExpectedException("\PHPUnit_Framework_Error");
1301     $this->assertFalse(rmdir("gs://bucket/dira/dirb"));
1302     $this->apiProxyMock->verify();
1303   }
1304
1305   public function testStreamCast() {
1306     $body = "Hello from PHP";
1307
1308     $this->expectFileReadRequest($body,
1309                                  0,
1310                                  CloudStorageReadClient::DEFAULT_READ_SIZE,
1311                                  null);
1312
1313     $valid_path = "gs://bucket/object_name.png";
1314     $this->setExpectedException(
1315         '\PHPUnit_Framework_Error',
1316         'gzopen(): cannot represent a stream of type user-space as a ' .
1317         'File Descriptor');
1318     $this->assertFalse(gzopen($valid_path, 'rb'));
1319     $this->apiProxyMock->verify();
1320   }
1321
1322   private function expectFileReadRequest($body,
1323                                          $start_byte,
1324                                          $length,
1325                                          $etag = null,
1326                                          $paritial_content = null) {
1327     $this->expectGetAccessTokenRequest(CloudStorageClient::READ_SCOPE);
1328
1329     assert($length > 0);
1330     $last_byte = $start_byte + $length - 1;
1331     $request_headers = [
1332         "Authorization" => "OAuth foo token",
1333         "Range" => sprintf("bytes=%d-%d", $start_byte, $last_byte),
1334     ];
1335
1336     if (isset($etag)) {
1337       $request_headers['If-Match'] = $etag;
1338     }
1339
1340     $request_headers["x-goog-api-version"] = 2;
1341
1342     $response_headers = [
1343         "ETag" => "deadbeef",
1344         "Content-Type" => "text/plain",
1345         "Last-Modified" => "Mon, 02 Jul 2012 01:41:01 GMT",
1346     ];
1347
1348     $response = $this->createSuccessfulGetHttpResponse($response_headers,
1349                                                        $body,
1350                                                        $start_byte,
1351                                                        $length,
1352                                                        $paritial_content);
1353
1354     $exected_url = self::makeCloudStorageObjectUrl("bucket",
1355                                                    "/object_name.png");
1356
1357     $this->expectHttpRequest($exected_url,
1358                              RequestMethod::GET,
1359                              $request_headers,
1360                              null,
1361                              $response);
1362   }
1363
1364   private function expectGetAccessTokenRequest($scope) {
1365     $req = new \google\appengine\GetAccessTokenRequest();
1366
1367     $req->addScope($scope);
1368
1369     $resp = new \google\appengine\GetAccessTokenResponse();
1370     $resp->setAccessToken('foo token');
1371     $resp->setExpirationTime(12345);
1372
1373     $this->apiProxyMock->expectCall('app_identity_service',
1374                                     'GetAccessToken',
1375                                     $req,
1376                                     $resp);
1377
1378     $this->mock_memcache->expects($this->at($this->mock_memcache_call_index++))
1379                         ->method('get')
1380                         ->with($this->stringStartsWith('_ah_app_identity'))
1381                         ->will($this->returnValue(false));
1382
1383     $this->mock_memcache->expects($this->at($this->mock_memcache_call_index++))
1384                         ->method('set')
1385                         ->with($this->stringStartsWith('_ah_app_identity'),
1386                                $this->anything(),
1387                                $this->anything(),
1388                                $this->anything())
1389                         ->will($this->returnValue(false));
1390   }
1391
1392   private function createSuccessfulGetHttpResponse($headers,
1393                                                    $body,
1394                                                    $start_byte,
1395                                                    $length,
1396                                                    $return_partial_content) {
1397     $total_body_length = strlen($body);
1398     $partial_content = false;
1399     $range_cannot_be_satisfied = false;
1400
1401     if ($total_body_length <= $start_byte) {
1402       $range_cannot_be_satisfied = true;
1403       $body = "<Message>The requested range cannot be satisfied.</Message>";
1404     } else {
1405       if ($start_byte != 0 || $length < $total_body_length) {
1406         $final_length = min($length, $total_body_length - $start_byte);
1407         $body = substr($body, $start_byte, $final_length);
1408         $partial_content = true;
1409       } else if ($return_partial_content) {
1410         $final_length = strlen($body);
1411         $partial_content = true;
1412       }
1413     }
1414
1415     $success_headers = [];
1416     if ($range_cannot_be_satisfied) {
1417       $status_code = HttpResponse::RANGE_NOT_SATISFIABLE;
1418       $success_headers["Content-Length"] = $total_body_length;
1419     } else if (!$partial_content) {
1420       $status_code = HttpResponse::OK;
1421       $success_headers["Content-Length"] = $total_body_length;
1422     } else {
1423       $status_code = HttpResponse::PARTIAL_CONTENT;
1424       $end_range = $start_byte + $final_length - 1;
1425       $success_headers["Content-Length"] = $final_length;
1426       $success_headers["Content-Range"] = sprintf("bytes %d-%d/%d",
1427                                                   $start_byte,
1428                                                   $end_range,
1429                                                   $total_body_length);
1430     }
1431
1432     return [
1433         'status_code' => $status_code,
1434         'headers' => array_merge($success_headers, $headers),
1435         'body' => $body,
1436     ];
1437   }
1438
1439   private function expectFileWriteStartRequest($content_type,
1440                                                $acl,
1441                                                $id,
1442                                                $url,
1443                                                $metadata = NULL) {
1444     $this->expectGetAccessTokenRequest(CloudStorageClient::WRITE_SCOPE);
1445     $upload_id =  "https://host/bucket/object.png?upload_id=" . $id;
1446     // The upload will start with a POST to acquire the upload ID.
1447     $request_headers = [
1448         "x-goog-resumable" => "start",
1449         "Authorization" => "OAuth foo token",
1450     ];
1451     if ($content_type != null) {
1452       $request_headers['Content-Type'] = $content_type;
1453     }
1454     if ($acl != null) {
1455       $request_headers['x-goog-acl'] = $acl;
1456     }
1457     if (isset($metadata)) {
1458       foreach ($metadata as $key => $value) {
1459         $request_headers["x-goog-meta-" . $key] = $value;
1460       }
1461     }
1462     $request_headers["x-goog-api-version"] = 2;
1463     $response = [
1464         'status_code' => 201,
1465         'headers' => [
1466             'Location' => $upload_id,
1467         ],
1468     ];
1469     $this->expectHttpRequest($url,
1470                              RequestMethod::POST,
1471                              $request_headers,
1472                              null,
1473                              $response);
1474   }
1475
1476   private function expectFileWriteContentRequest($url,
1477                                                  $upload_id,
1478                                                  $data,
1479                                                  $start_byte,
1480                                                  $end_byte,
1481                                                  $complete) {
1482     // The upload will be completed with a PUT with the final length
1483     $this->expectGetAccessTokenRequest(CloudStorageClient::WRITE_SCOPE);
1484     // If start byte is null then we assume that this is a PUT with no content,
1485     // and the end_byte contains the length of the data to write.
1486     if (is_null($start_byte)) {
1487       $range = sprintf("bytes */%d", $end_byte);
1488       $status_code = HttpResponse::OK;
1489       $body = null;
1490     } else {
1491       $length = $end_byte - $start_byte + 1;
1492       if ($complete) {
1493         $total_len = $end_byte + 1;
1494         $range = sprintf("bytes %d-%d/%d", $start_byte, $end_byte, $total_len);
1495         $status_code = HttpResponse::OK;
1496       } else {
1497         $range = sprintf("bytes %d-%d/*", $start_byte, $end_byte);
1498         $status_code = HttpResponse::RESUME_INCOMPLETE;
1499       }
1500       $body = substr($data, $start_byte, $length);
1501     }
1502     $request_headers = [
1503         "Authorization" => "OAuth foo token",
1504         "Content-Range" => $range,
1505         "x-goog-api-version" => 2,
1506     ];
1507     $response = [
1508         'status_code' => $status_code,
1509         'headers' => [
1510         ],
1511     ];
1512     $expected_url = $url . "?upload_id=" . $upload_id;
1513     $this->expectHttpRequest($expected_url,
1514                              RequestMethod::PUT,
1515                              $request_headers,
1516                              $body,
1517                              $response);
1518   }
1519
1520   private function expectHttpRequest($url, $method, $headers, $body, $result) {
1521     $req = new \google\appengine\URLFetchRequest();
1522     $req->setUrl($url);
1523     $req->setMethod($method);
1524     $req->setMustValidateServerCertificate(true);
1525
1526     foreach($headers as $k => $v) {
1527       $h = $req->addHeader();
1528       $h->setKey($k);
1529       $h->setValue($v);
1530     }
1531
1532     if (isset($body)) {
1533       $req->setPayload($body);
1534     }
1535
1536     $resp = new \google\appengine\URLFetchResponse();
1537
1538     $resp->setStatusCode($result['status_code']);
1539     foreach($result['headers'] as $k => $v) {
1540       $h = $resp->addHeader();
1541       $h->setKey($k);
1542       $h->setValue($v);
1543     }
1544     if (isset($result['body'])) {
1545       $resp->setContent($result['body']);
1546     }
1547
1548     $this->apiProxyMock->expectCall('urlfetch',
1549                                     'Fetch',
1550                                     $req,
1551                                     $resp);
1552   }
1553
1554   private function expectIsWritableMemcacheLookup($key_found, $result) {
1555     if ($key_found) {
1556       $lookup_result = ['is_writable' => $result];
1557     } else {
1558       $lookup_result = false;
1559     }
1560
1561     $this->mock_memcache->expects($this->at($this->mock_memcache_call_index++))
1562                         ->method('get')
1563                         ->with($this->stringStartsWith(
1564                             '_ah_gs_write_bucket_cache_'))
1565                         ->will($this->returnValue($lookup_result));
1566   }
1567
1568   private function expectIsWritableMemcacheSet($value) {
1569     $this->mock_memcache->expects($this->at($this->mock_memcache_call_index++))
1570         ->method('set')
1571         ->with($this->stringStartsWith('_ah_gs_write_bucket_cache_'),
1572                ['is_writable' => $value],
1573                null,
1574                CloudStorageClient::DEFAULT_WRITABLE_CACHE_EXPIRY_SECONDS)
1575         ->will($this->returnValue(false));
1576   }
1577
1578   private function makeCloudStorageObjectUrl($bucket = "bucket",
1579                                              $object = "/object.png") {
1580     return CloudStorageClient::createObjectUrl($bucket, $object);
1581   }
1582
1583   private function getStandardRequestHeaders() {
1584     return [
1585         "Authorization" => "OAuth foo token",
1586         "x-goog-api-version" => 2,
1587     ];
1588   }
1589
1590   private function makeGetBucketXmlResponse($prefix,
1591                                             $contents_array,
1592                                             $next_marker = null,
1593                                             $common_prefix_array = null) {
1594     $result = "<?xml version='1.0' encoding='UTF-8'?>
1595         <ListBucketResult xmlns='http://doc.s3.amazonaws.com/2006-03-01'>
1596         <Name>sjl-test</Name>
1597         <Prefix>" . $prefix . "</Prefix>
1598         <Marker></Marker>";
1599     if (isset($next_marker)) {
1600       $result .= "<NextMarker>" . $next_marker . "</NextMarker>";
1601     }
1602     $result .= "<Delimiter>/</Delimiter>
1603         <IsTruncated>false</IsTruncated>";
1604
1605     foreach($contents_array as $content) {
1606       $result .= '<Contents>';
1607       if (is_string($content)) {
1608         $result .= '<Key>' . $content . '</Key>';
1609       } else {
1610         $result .= '<Key>' . $content['key'] . '</Key>';
1611         $result .= '<Size>' . $content['size'] . '</Size>';
1612         $result .= '<LastModified>' . $content['mtime'] . '</LastModified>';
1613       }
1614       $result .= '</Contents>';
1615     }
1616     if (isset($common_prefix_array)) {
1617       foreach($common_prefix_array as $common_prefix) {
1618         $result .= '<CommonPrefixes>';
1619         $result .= '<Prefix>' . $common_prefix . '</Prefix>';
1620         $result .= '</CommonPrefixes>';
1621       }
1622     }
1623     $result .= "</ListBucketResult>";
1624     return $result;
1625   }
1626 }
1627
1628 }  // namespace google\appengine\ext\cloud_storage_streams;
1629