botlistprojects/botprojects_lib/java/src/S3Test.java

   1 //  This software code is made available "AS IS" without warranties of any
   2 //  kind.  You may copy, display, modify and redistribute the software
   3 //  code either by itself or as incorporated into your code; provided that
   4 //  you do not remove any proprietary notices.  Your use of this software
   5 //  code is at your own risk and you waive any claim against Amazon
   6 //  Digital Services, Inc. or its affiliates with respect to your use of
   7 //  this software code. (c) 2006-2007 Amazon Digital Services, Inc. or its
   8 //  affiliates.
   9
  10 import java.io.IOException;
  11 import java.net.HttpURLConnection;
  12 import java.net.MalformedURLException;
  13 import java.net.URL;
  14 import java.util.Arrays;
  15 import java.util.HashMap;
  16 import java.util.Iterator;
  17 import java.util.List;
  18 import java.util.Map;
  19
  20 import com.amazon.s3.AWSAuthConnection;
  21 import com.amazon.s3.CallingFormat;
  22 import com.amazon.s3.GetResponse;
  23 import com.amazon.s3.ListAllMyBucketsResponse;
  24 import com.amazon.s3.ListBucketResponse;
  25 import com.amazon.s3.ListEntry;
  26 import com.amazon.s3.QueryStringAuthGenerator;
  27 import com.amazon.s3.Response;
  28 import com.amazon.s3.S3Object;
  29 import com.amazon.s3.Utils;
  30
  31 public class S3Test {
  32     static final String awsAccessKeyId = "<INSERT YOUR AWS ACCESS KEY ID HERE>";
  33     static final String awsSecretAccessKey = "<INSERT YOUR AWS SECRET ACCESS KEY HERE>";
  34
  35     // for subdomains (bucket.s3.amazonaws.com),
  36     // the bucket name must be lowercase since DNS is case-insensitive
  37     static final String bucketName = awsAccessKeyId.toLowerCase() + "-test-bucket";
  38     static int assertionCount = 0;
  39
  40     static final int UnspecifiedMaxKeys = -1;
  41
  42     public static void main(String args[]) throws Exception {
  43         if (awsAccessKeyId.startsWith("<INSERT")) {
  44             System.err.println("Please examine S3Test.java and update it with your credentials");
  45             System.exit(-1);
  46         }
  47
  48         // test all operation for both regular and vanity domains
  49         // regular: http://s3.amazonaws.com/key
  50         // subdomain: http://bucket.s3.amazonaws.com/key
  51         // testing pure vanity domains (http://<vanity domain>/key) is not covered here
  52         // but is possible with some assitional setup
  53         test(CallingFormat.getSubdomainCallingFormat(), AWSAuthConnection.LOCATION_DEFAULT, true, Utils.DEFAULT_HOST);
  54         test(CallingFormat.getPathCallingFormat(), AWSAuthConnection.LOCATION_DEFAULT, true, Utils.DEFAULT_HOST);
  55         test(CallingFormat.getSubdomainCallingFormat(), AWSAuthConnection.LOCATION_EU, true, Utils.DEFAULT_HOST);
  56     }
  57
  58     private static void test(CallingFormat format, String location, boolean secure, String server) throws Exception
  59     {
  60         assertionCount = 0;
  61         System.out.println((secure ? "http" : "https") + " / " + server + " / " +
  62                 ((location == null) ? "<no-location>" : location) + " / " +
  63                 format.getClass().getName());
  64
  65         AWSAuthConnection conn = new AWSAuthConnection(awsAccessKeyId, awsSecretAccessKey, secure, server, format);
  66         QueryStringAuthGenerator generator =
  67             new QueryStringAuthGenerator(awsAccessKeyId, awsSecretAccessKey, secure, server, format);
  68
  69         Response response = conn.createBucket(bucketName, location, null);
  70         assertEquals(
  71                 "couldn't create bucket",
  72                 HttpURLConnection.HTTP_OK,
  73                 response.connection.getResponseCode());
  74
  75         ListBucketResponse listBucketResponse = conn.listBucket(bucketName, null, null, null, null);
  76         assertEquals(
  77                 "couldn't get list",
  78                 HttpURLConnection.HTTP_OK,
  79                 listBucketResponse.connection.getResponseCode());
  80         assertEquals("list wasn't empty", 0, listBucketResponse.entries.size());
  81         verifyBucketResponseParameters(listBucketResponse, bucketName, "", "", UnspecifiedMaxKeys, null, false, null);
  82
  83         // start delimiter tests
  84
  85         final String text = "this is a test";
  86         final String key = "example.txt";
  87         final String innerKey = "test/inner.txt";
  88         final String lastKey = "z-last-key.txt";
  89
  90         response = conn.put(bucketName, key, new S3Object(text.getBytes(), null), null);
  91         assertEquals(
  92                 "couldn't put simple object",
  93                 HttpURLConnection.HTTP_OK,
  94                 response.connection.getResponseCode());
  95
  96         response = conn.put(bucketName, innerKey, new S3Object(text.getBytes(), null), null);
  97         assertEquals(
  98                 "couldn't put simple object",
  99                 HttpURLConnection.HTTP_OK,
 100                 response.connection.getResponseCode());
 101
 102         response = conn.put(bucketName, lastKey, new S3Object(text.getBytes(), null), null);
 103         assertEquals(
 104                 "couldn't put simple object",
 105                 HttpURLConnection.HTTP_OK,
 106                 response.connection.getResponseCode());
 107
 108         // plain list
 109         listBucketResponse = conn.listBucket(bucketName, null, null, null, null);
 110         assertEquals(
 111                 "couldn't get list",
 112                 listBucketResponse.connection.getResponseCode(),
 113                 HttpURLConnection.HTTP_OK);
 114         assertEquals("Unexpected list size", 3, listBucketResponse.entries.size());
 115         assertEquals("Unexpected common prefix size", 0, listBucketResponse.commonPrefixEntries.size());
 116         verifyBucketResponseParameters(listBucketResponse, bucketName, "", "", UnspecifiedMaxKeys, null, false, null);
 117
 118         // root "directory"
 119         listBucketResponse = conn.listBucket(bucketName, null, null, null, "/", null);
 120         assertEquals(
 121                 "couldn't get list",
 122                 HttpURLConnection.HTTP_OK,
 123                 listBucketResponse.connection.getResponseCode());
 124         assertEquals("Unexpected list size", 2, listBucketResponse.entries.size());
 125         assertEquals("Unexpected common prefix size", 1, listBucketResponse.commonPrefixEntries.size());
 126         verifyBucketResponseParameters(listBucketResponse, bucketName, "", "", UnspecifiedMaxKeys, "/", false, null);
 127
 128         // root "directory" with a max-keys of "1"
 129         listBucketResponse = conn.listBucket(bucketName, null, null, new Integer( 1 ), "/", null);
 130         assertEquals(
 131                 "couldn't get list",
 132                 HttpURLConnection.HTTP_OK,
 133                 listBucketResponse.connection.getResponseCode());
 134         assertEquals("Unexpected list size", 1, listBucketResponse.entries.size());
 135         assertEquals("Unexpected common prefix size", 0, listBucketResponse.commonPrefixEntries.size());
 136         verifyBucketResponseParameters(listBucketResponse, bucketName, "", "", 1, "/", true, "example.txt");
 137
 138         // root "directory" with a max-keys of "2"
 139         listBucketResponse = conn.listBucket(bucketName, null, null, new Integer( 2 ), "/", null);
 140         assertEquals(
 141                 "couldn't get list",
 142                 HttpURLConnection.HTTP_OK,
 143                 listBucketResponse.connection.getResponseCode());
 144         assertEquals("Unexpected list size", 1, listBucketResponse.entries.size());
 145         assertEquals("Unexpected common prefix size", 1, listBucketResponse.commonPrefixEntries.size());
 146         verifyBucketResponseParameters(listBucketResponse, bucketName, "", "", 2, "/", true, "test/");
 147         String marker = listBucketResponse.nextMarker;
 148         listBucketResponse = conn.listBucket(bucketName, null, marker, new Integer( 2 ), "/", null);
 149         assertEquals(
 150                 "couldn't get list",
 151                 HttpURLConnection.HTTP_OK,
 152                 listBucketResponse.connection.getResponseCode());
 153         assertEquals("Unexpected list size", 1, listBucketResponse.entries.size());
 154         assertEquals("Unexpected common prefix size", 0, listBucketResponse.commonPrefixEntries.size());
 155         verifyBucketResponseParameters(listBucketResponse, bucketName, "", marker, 2, "/", false, null);
 156
 157         // test "directory"
 158         listBucketResponse = conn.listBucket(bucketName, "test/", null, null, "/", null);
 159         assertEquals(
 160                 "couldn't get list",
 161                 HttpURLConnection.HTTP_OK,
 162                 listBucketResponse.connection.getResponseCode());
 163         assertEquals("Unexpected list size", 1, listBucketResponse.entries.size());
 164         assertEquals("Unexpected common prefix size", 0, listBucketResponse.commonPrefixEntries.size());
 165         verifyBucketResponseParameters(listBucketResponse, bucketName, "test/", "", UnspecifiedMaxKeys, "/", false, null);
 166
 167         // remove innerkey
 168         response = conn.delete(bucketName, innerKey, null);
 169         assertEquals(
 170                 "couldn't delete entry",
 171                 HttpURLConnection.HTTP_NO_CONTENT,
 172                 response.connection.getResponseCode());
 173
 174         // remove last key
 175         response = conn.delete(bucketName, lastKey, null);
 176         assertEquals(
 177                 "couldn't delete entry",
 178                 HttpURLConnection.HTTP_NO_CONTENT,
 179                 response.connection.getResponseCode());
 180
 181
 182         // end delimiter tests
 183
 184         response = conn.put(bucketName, key, new S3Object(text.getBytes(), null), null);
 185         assertEquals(
 186                 "couldn't reput simple object",
 187                 HttpURLConnection.HTTP_OK,
 188                 response.connection.getResponseCode());
 189
 190         Map metadata = new HashMap();
 191         metadata.put("title", Arrays.asList(new String[] { "title" }));
 192         response = conn.put(bucketName, key, new S3Object(text.getBytes(), metadata), null);
 193         assertEquals(
 194                 "couldn't put complex object",
 195                 HttpURLConnection.HTTP_OK,
 196                 response.connection.getResponseCode());
 197
 198         GetResponse getResponse = conn.get(bucketName, key, null);
 199         assertEquals(
 200                 "couldn't get object",
 201                 getResponse.connection.getResponseCode(),
 202                 HttpURLConnection.HTTP_OK);
 203         assertEquals("didn't get the right data back", text.getBytes(), getResponse.object.data);
 204         assertEquals("didn't get the right metadata back", 1, getResponse.object.metadata.size());
 205         assertEquals(
 206                 "didn't get the right metadata back",
 207                 1,
 208                 ((List)getResponse.object.metadata.get("title")).size());
 209         assertEquals(
 210                 "didn't get the right metadata back",
 211                 "title",
 212                 ((List)getResponse.object.metadata.get("title")).get(0));
 213         assertEquals(
 214                 "didn't get the right content-length",
 215                 ""+text.length(),
 216                 getResponse.connection.getHeaderField("Content-Length"));
 217
 218
 219         Map metadataWithSpaces = new HashMap();
 220         String titleWithSpaces = " \t  title with leading and trailing spaces    ";
 221         metadataWithSpaces.put("title", Arrays.asList(new String[] { titleWithSpaces }));
 222         response = conn.put(bucketName, key, new S3Object(text.getBytes(), metadataWithSpaces), null);
 223         assertEquals(
 224                 "couldn't put metadata with leading and trailing spaces",
 225                 HttpURLConnection.HTTP_OK,
 226                 response.connection.getResponseCode());
 227
 228         getResponse = conn.get(bucketName, key, null);
 229         assertEquals(
 230                 "couldn't get object",
 231                 HttpURLConnection.HTTP_OK,
 232                 getResponse.connection.getResponseCode());
 233         assertEquals("didn't get the right metadata back", getResponse.object.metadata.size(), 1);
 234         assertEquals(
 235                 "didn't get the right metadata back",
 236                 1,
 237                 ((List)getResponse.object.metadata.get("title")).size());
 238         assertEquals(
 239                 "didn't get the right metadata back",
 240                 titleWithSpaces.trim(),
 241                 ((List)getResponse.object.metadata.get("title")).get(0));
 242
 243         String weirdKey = "&=//%# ++++";
 244         response = conn.put(bucketName, weirdKey, new S3Object(text.getBytes(), null), null);
 245         assertEquals(
 246                 "couldn't put weird key",
 247                 HttpURLConnection.HTTP_OK,
 248                 response.connection.getResponseCode());
 249
 250         getResponse = conn.get(bucketName, weirdKey, null);
 251         assertEquals(
 252                 "couldn't get weird key",
 253                 HttpURLConnection.HTTP_OK,
 254                 getResponse.connection.getResponseCode());
 255
 256         // start acl test
 257
 258         getResponse = conn.getACL(bucketName, key, null);
 259         assertEquals(
 260                 "couldn't get acl",
 261                 HttpURLConnection.HTTP_OK,
 262                 getResponse.connection.getResponseCode());
 263
 264         byte[] acl = getResponse.object.data;
 265
 266         response = conn.putACL(bucketName, key, new String(acl), null);
 267         assertEquals(
 268                 "couldn't put acl",
 269                 HttpURLConnection.HTTP_OK,
 270                 response.connection.getResponseCode());
 271
 272         getResponse = conn.getBucketACL(bucketName, null);
 273         assertEquals(
 274                 "couldn't get bucket acl",
 275                 HttpURLConnection.HTTP_OK,
 276                 getResponse.connection.getResponseCode());
 277
 278         byte[] bucketACL = getResponse.object.data;
 279
 280         response = conn.putBucketACL(bucketName, new String(bucketACL), null);
 281         assertEquals(
 282                 "couldn't put bucket acl",
 283                 HttpURLConnection.HTTP_OK,
 284                 response.connection.getResponseCode());
 285
 286         // end acl test
 287
 288         // bucket logging tests
 289         getResponse = conn.getBucketLogging(bucketName, null);
 290         assertEquals(
 291                 "couldn't get bucket logging config",
 292                 HttpURLConnection.HTTP_OK,
 293                 getResponse.connection.getResponseCode());
 294
 295         byte[] bucketLogging = getResponse.object.data;
 296
 297         response = conn.putBucketLogging(bucketName, new String(bucketLogging), null);
 298         assertEquals(
 299                 "couldn't put bucket logging config",
 300                 HttpURLConnection.HTTP_OK,
 301                 response.connection.getResponseCode());
 302
 303         // end bucket logging tests
 304
 305         listBucketResponse = conn.listBucket(bucketName, null, null, null, null);
 306         assertEquals(
 307                 "couldn't list bucket",
 308                 HttpURLConnection.HTTP_OK,
 309                 listBucketResponse.connection.getResponseCode());
 310         List entries = listBucketResponse.entries;
 311         assertEquals("didn't get back the right number of entries", 2, entries.size());
 312         // depends on weirdKey < $key
 313         assertEquals("first key isn't right", weirdKey, ((ListEntry)entries.get(0)).key);
 314         assertEquals("second key isn't right", key, ((ListEntry)entries.get(1)).key);
 315         verifyBucketResponseParameters(listBucketResponse, bucketName, "", "", UnspecifiedMaxKeys, null, false, null);
 316
 317         listBucketResponse = conn.listBucket(bucketName, null, null, new Integer(1), null);
 318         assertEquals(
 319                 "couldn't list bucket",
 320                 HttpURLConnection.HTTP_OK,
 321                 listBucketResponse.connection.getResponseCode());
 322         assertEquals(
 323                 "didn't get back the right number of entries",
 324                 1,
 325                 listBucketResponse.entries.size());
 326         verifyBucketResponseParameters(listBucketResponse, bucketName, "", "", 1, null, true, null);
 327
 328         for (Iterator it = entries.iterator(); it.hasNext(); ) {
 329             ListEntry entry = (ListEntry)it.next();
 330             response = conn.delete(bucketName, entry.key, null);
 331             assertEquals(
 332                     "couldn't delete entry",
 333                     HttpURLConnection.HTTP_NO_CONTENT,
 334                     response.connection.getResponseCode());
 335         }
 336
 337         ListAllMyBucketsResponse listAllMyBucketsResponse = conn.listAllMyBuckets(null);
 338         assertEquals(
 339                 "couldn't list all my buckets",
 340                 HttpURLConnection.HTTP_OK,
 341                 listAllMyBucketsResponse.connection.getResponseCode());
 342         List buckets = listAllMyBucketsResponse.entries;
 343
 344         response = conn.deleteBucket(bucketName, null);
 345         assertEquals(
 346                 "couldn't delete bucket",
 347                 HttpURLConnection.HTTP_NO_CONTENT,
 348                 response.connection.getResponseCode());
 349
 350         listAllMyBucketsResponse = conn.listAllMyBuckets(null);
 351         assertEquals(
 352                 "couldn't list all my buckets",
 353                 HttpURLConnection.HTTP_OK,
 354                 listAllMyBucketsResponse.connection.getResponseCode());
 355         assertEquals(
 356                 "bucket count is incorrect",
 357                 buckets.size() - 1,
 358                 listAllMyBucketsResponse.entries.size());
 359
 360         checkURL(
 361                 generator.createBucket(bucketName, null),
 362                 "PUT",
 363                 HttpURLConnection.HTTP_OK,
 364                 "couldn't create bucket");
 365         checkURL(
 366                 generator.put(bucketName, key, new S3Object("test data".getBytes(), null), null),
 367                 "PUT",
 368                 HttpURLConnection.HTTP_OK,
 369                 "put object",
 370                 "test data");
 371         checkURL(
 372                 generator.get(bucketName, key, null),
 373                 "GET",
 374                 HttpURLConnection.HTTP_OK,
 375                 "get object");
 376         checkURL(
 377                 generator.listBucket(bucketName, null, null, null, null),
 378                 "GET",
 379                 HttpURLConnection.HTTP_OK,
 380                 "list bucket");
 381         checkURL(
 382                 generator.listAllMyBuckets(null),
 383                 "GET",
 384                 HttpURLConnection.HTTP_OK,
 385                 "list all my buckets");
 386         checkURL(
 387                 generator.getACL(bucketName, key, null),
 388                 "GET",
 389                 HttpURLConnection.HTTP_OK,
 390                 "get acl");
 391         checkURL(
 392                 generator.putACL(bucketName, key, new String(acl), null),
 393                 "PUT",
 394                 HttpURLConnection.HTTP_OK,
 395                 "put acl",
 396                 new String(acl));
 397         checkURL(
 398                 generator.getBucketACL(bucketName, null),
 399                 "GET",
 400                 HttpURLConnection.HTTP_OK,
 401                 "get bucket acl");
 402         checkURL(
 403                 generator.putBucketACL(bucketName, new String(bucketACL), null),
 404                 "PUT",
 405                 HttpURLConnection.HTTP_OK,
 406                 "put bucket acl",
 407                 new String(bucketACL));
 408         checkURL(
 409                 generator.getBucketLogging(bucketName, null),
 410                 "GET",
 411                 HttpURLConnection.HTTP_OK,
 412                 "get bucket logging");
 413         checkURL(
 414                 generator.putBucketLogging(bucketName, new String(bucketLogging), null),
 415                 "PUT",
 416                 HttpURLConnection.HTTP_OK,
 417                 "put bucket logging",
 418                 new String(bucketLogging));
 419         checkURL(
 420                 generator.delete(bucketName, key, null),
 421                 "DELETE",
 422                 HttpURLConnection.HTTP_NO_CONTENT,
 423                 "delete object");
 424         checkURL(
 425                 generator.deleteBucket(bucketName, null),
 426                 "DELETE",
 427                 HttpURLConnection.HTTP_NO_CONTENT,
 428                 "delete bucket");
 429
 430         System.out.println("OK (" + assertionCount + " tests passed)");
 431     }
 432
 433     private static void verifyBucketResponseParameters( ListBucketResponse listBucketResponse,
 434                                                            String bucketName, String prefix, String marker,
 435                                                            int maxKeys, String delimiter, boolean isTruncated,
 436                                                            String nextMarker ) {
 437         assertEquals("Bucket name should match.", bucketName, listBucketResponse.name);
 438         assertEquals("Bucket prefix should match.", prefix, listBucketResponse.prefix);
 439         assertEquals("Bucket marker should match.", marker, listBucketResponse.marker);
 440         assertEquals("Bucket delimiter should match.", delimiter, listBucketResponse.delimiter);
 441         if ( UnspecifiedMaxKeys != maxKeys ) {
 442             assertEquals("Bucket max-keys should match.", maxKeys, listBucketResponse.maxKeys);
 443         }
 444         assertEquals("Bucket should not be truncated.", isTruncated, listBucketResponse.isTruncated);
 445         assertEquals("Bucket nextMarker should match.", nextMarker, listBucketResponse.nextMarker);
 446     }
 447
 448
 449     private static void assertEquals(String message, int expected, int actual) {
 450         assertionCount++;
 451         if (expected != actual) {
 452             throw new RuntimeException(message + ": expected " + expected + " but got " + actual);
 453         }
 454     }
 455
 456     private static void assertEquals(String message, byte[] expected, byte[] actual) {
 457         assertionCount++;
 458         if (! Arrays.equals(expected, actual)) {
 459             throw new RuntimeException(
 460                     message +
 461                     ": expected " +
 462                     new String(expected) +
 463                     " but got " +
 464                     new String(actual));
 465         }
 466     }
 467
 468     private static void assertEquals(String message, Object expected, Object actual) {
 469         assertionCount++;
 470         if (expected != actual && (actual == null || ! actual.equals(expected))) {
 471             throw new RuntimeException(message + ": expected " + expected + " but got " + actual);
 472         }
 473     }
 474
 475     private static void assertEquals(String message, boolean expected, boolean actual) {
 476         assertionCount++;
 477         if (expected != actual) {
 478             throw new RuntimeException(message + ": expected " + expected + " but got " + actual);
 479         }
 480     }
 481
 482     private static void checkURL(String url, String method, int code, String message)
 483         throws MalformedURLException, IOException
 484     {
 485         checkURL(url, method, code, message, null);
 486     }
 487
 488     private static void checkURL(String url, String method, int code, String message, String data)
 489         throws MalformedURLException, IOException
 490     {
 491         if (data == null) data = "";
 492
 493         HttpURLConnection connection = (HttpURLConnection)new URL(url).openConnection();
 494         connection.setRequestMethod(method);
 495         if ("PUT".equals(method)) {
 496             connection.setRequestProperty("Content-Length", ""+data.getBytes().length);
 497             connection.setDoOutput(true);
 498             connection.getOutputStream().write(data.getBytes());
 499         } else {
 500             // HttpURLConnection auto populates Content-Type, which we don't want here
 501             connection.setRequestProperty("Content-Type", "");
 502         }
 503
 504         assertEquals(message, code, connection.getResponseCode());
 505     }
 506 }