test/unit/voldemort/coordinator/CoordinatorRestAPITest.java

   1 /*
   2  * Copyright 2013 LinkedIn, Inc
   3  *
   4  * Licensed under the Apache License, Version 2.0 (the "License"); you may not
   5  * use this file except in compliance with the License. You may obtain a copy of
   6  * the License at
   7  *
   8  * http://www.apache.org/licenses/LICENSE-2.0
   9  *
  10  * Unless required by applicable law or agreed to in writing, software
  11  * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  12  * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  13  * License for the specific language governing permissions and limitations under
  14  * the License.
  15  */
  16
  17 package voldemort.coordinator;
  18
  19 import static org.junit.Assert.assertEquals;
  20 import static org.junit.Assert.fail;
  21
  22 import java.io.OutputStream;
  23 import java.net.HttpURLConnection;
  24 import java.net.URL;
  25 import java.util.ArrayList;
  26 import java.util.HashMap;
  27 import java.util.List;
  28 import java.util.Map;
  29 import java.util.Properties;
  30
  31 import javax.mail.internet.MimeBodyPart;
  32 import javax.mail.internet.MimeMultipart;
  33 import javax.mail.util.ByteArrayDataSource;
  34
  35 import org.apache.commons.codec.binary.Base64;
  36 import org.junit.After;
  37 import org.junit.Before;
  38 import org.junit.Test;
  39
  40 import voldemort.ServerTestUtils;
  41 import voldemort.rest.RestMessageHeaders;
  42 import voldemort.rest.RestUtils;
  43 import voldemort.rest.coordinator.CoordinatorConfig;
  44 import voldemort.rest.coordinator.CoordinatorService;
  45 import voldemort.server.VoldemortServer;
  46 import voldemort.store.socket.SocketStoreFactory;
  47 import voldemort.store.socket.clientrequest.ClientRequestExecutorPool;
  48 import voldemort.versioning.Occurred;
  49 import voldemort.versioning.VectorClock;
  50
  51 public class CoordinatorRestAPITest {
  52
  53     private VoldemortServer[] servers;
  54     public static String socketUrl = "";
  55     private static final String STORE_NAME = "slow-store-test";
  56     private static final String STORES_XML = "test/common/voldemort/config/single-slow-store.xml";
  57     private static final String FAT_CLIENT_CONFIG_FILE_PATH = "test/common/voldemort/config/fat-client-config.avro";
  58     private final SocketStoreFactory socketStoreFactory = new ClientRequestExecutorPool(2,
  59                                                                                         10000,
  60                                                                                         100000,
  61                                                                                         32 * 1024);
  62     private CoordinatorService coordinator = null;
  63     private final String coordinatorURL = "http://localhost:8080";
  64
  65     private class TestVersionedValue {
  66
  67         private String value;
  68         private VectorClock vc;
  69
  70         public TestVersionedValue(String val, VectorClock vc) {
  71             this.setValue(val);
  72             this.setVc(vc);
  73         }
  74
  75         public String getValue() {
  76             return value;
  77         }
  78
  79         public void setValue(String value) {
  80             this.value = value;
  81         }
  82
  83         public VectorClock getVc() {
  84             return vc;
  85         }
  86
  87         public void setVc(VectorClock vc) {
  88             this.vc = vc;
  89         }
  90     }
  91
  92     @Before
  93     public void setUp() throws Exception {
  94         int numServers = 1;
  95         servers = new VoldemortServer[numServers];
  96         int partitionMap[][] = { { 0, 2, 4, 6, 1, 3, 5, 7 } };
  97         Properties props = new Properties();
  98         props.setProperty("storage.configs",
  99                           "voldemort.store.bdb.BdbStorageConfiguration,voldemort.store.slow.SlowStorageConfiguration");
 100         props.setProperty("testing.slow.queueing.get.ms", "4");
 101         props.setProperty("testing.slow.queueing.getall.ms", "4");
 102         props.setProperty("testing.slow.queueing.put.ms", "4");
 103         props.setProperty("testing.slow.queueing.delete.ms", "4");
 104
 105         ServerTestUtils.startVoldemortCluster(numServers,
 106                                               servers,
 107                                               partitionMap,
 108                                               socketStoreFactory,
 109                                               true, // useNio
 110                                               null,
 111                                               STORES_XML,
 112                                               props);
 113
 114         CoordinatorConfig config = new CoordinatorConfig();
 115         List<String> bootstrapUrls = new ArrayList<String>();
 116         socketUrl = servers[0].getIdentityNode().getSocketUrl().toString();
 117         bootstrapUrls.add(socketUrl);
 118
 119         System.out.println("\n\n************************ Starting the Coordinator *************************");
 120
 121         config.setBootstrapURLs(bootstrapUrls);
 122         config.setFatClientConfigPath(FAT_CLIENT_CONFIG_FILE_PATH);
 123
 124         this.coordinator = new CoordinatorService(config);
 125         if(!this.coordinator.isStarted()) {
 126             this.coordinator.start();
 127         }
 128     }
 129
 130     @After
 131     public void tearDown() throws Exception {
 132         if(this.socketStoreFactory != null) {
 133             this.socketStoreFactory.close();
 134         }
 135
 136         if(this.coordinator != null && this.coordinator.isStarted()) {
 137             this.coordinator.stop();
 138         }
 139     }
 140
 141     public static enum ValueType {
 142         ALPHA,
 143         ALPHANUMERIC,
 144         NUMERIC
 145     }
 146
 147     public static String generateRandomString(int length, ValueType type) {
 148
 149         StringBuffer buffer = new StringBuffer();
 150         String characters = "";
 151
 152         switch(type) {
 153
 154             case ALPHA:
 155                 characters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ";
 156                 break;
 157
 158             case ALPHANUMERIC:
 159                 characters = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890";
 160                 break;
 161
 162             case NUMERIC:
 163                 characters = "1234567890";
 164                 break;
 165         }
 166
 167         int charactersLength = characters.length();
 168
 169         for(int i = 0; i < length; i++) {
 170             double index = Math.random() * charactersLength;
 171             buffer.append(characters.charAt((int) index));
 172         }
 173         return buffer.toString();
 174     }
 175
 176     private VectorClock doPut(String key, String payload, VectorClock vc) {
 177         return doPut(key, payload, vc, null);
 178     }
 179
 180     private VectorClock doPut(String key,
 181                               String payload,
 182                               VectorClock vc,
 183                               Map<String, Object> options) {
 184         VectorClock successfulPutVC = null;
 185         int expectedResponseCode = 201;
 186         try {
 187             // Create the right URL and Http connection
 188             HttpURLConnection conn = null;
 189             String base64Key = new String(Base64.encodeBase64(key.getBytes()));
 190             URL url = new URL(this.coordinatorURL + "/" + STORE_NAME + "/" + base64Key);
 191             conn = (HttpURLConnection) url.openConnection();
 192
 193             // Set the right headers
 194             conn.setRequestMethod("POST");
 195             conn.setDoOutput(true);
 196             conn.setDoInput(true);
 197             conn.setRequestProperty("Content-Type", "binary");
 198             conn.setRequestProperty("Content-Length", "" + payload.length());
 199             conn.setRequestProperty(RestMessageHeaders.X_VOLD_REQUEST_TIMEOUT_MS, "1000");
 200             conn.setRequestProperty(RestMessageHeaders.X_VOLD_REQUEST_ORIGIN_TIME_MS,
 201                                     Long.toString(System.currentTimeMillis()));
 202
 203             // options
 204             if(options != null) {
 205                 if(options.get("timeout") != null && options.get("timeout") instanceof String) {
 206                     conn.setRequestProperty(RestMessageHeaders.X_VOLD_REQUEST_TIMEOUT_MS,
 207                                             (String) options.get("timeout"));
 208                 }
 209                 if(options.get("responseCode") != null
 210                    && options.get("responseCode") instanceof Integer) {
 211                     expectedResponseCode = (Integer) options.get("responseCode");
 212                 }
 213             }
 214
 215             if(vc != null) {
 216                 String eTag = RestUtils.getSerializedVectorClock(vc);
 217                 conn.setRequestProperty("ETag", eTag);
 218             }
 219
 220             // Write the payload
 221             OutputStream out = conn.getOutputStream();
 222             out.write(payload.getBytes());
 223             out.close();
 224
 225             // Check for the right response code
 226
 227             if(conn.getResponseCode() != expectedResponseCode) {
 228                 System.err.println("Illegal response during PUT : " + conn.getResponseMessage());
 229                 fail("Incorrect response received for a HTTP put request :"
 230                      + conn.getResponseCode());
 231             }
 232
 233         } catch(Exception e) {
 234             e.printStackTrace();
 235             fail("Error in sending the REST request");
 236         }
 237
 238         return successfulPutVC;
 239     }
 240
 241     private boolean doDelete(String key) {
 242         return doDelete(key, null);
 243     }
 244
 245     private boolean doDelete(String key, Map<String, Object> options) {
 246         int expectedResponseCode = 204;
 247         try {
 248
 249             // Create the right URL and Http connection
 250             HttpURLConnection conn = null;
 251             String base64Key = new String(Base64.encodeBase64(key.getBytes()));
 252             URL url = new URL(this.coordinatorURL + "/" + STORE_NAME + "/" + base64Key);
 253             conn = (HttpURLConnection) url.openConnection();
 254
 255             // Set the right headers
 256             conn.setRequestMethod("DELETE");
 257             conn.setDoInput(true);
 258             conn.setRequestProperty(RestMessageHeaders.X_VOLD_REQUEST_TIMEOUT_MS, "1000");
 259             conn.setRequestProperty(RestMessageHeaders.X_VOLD_REQUEST_ORIGIN_TIME_MS,
 260                                     Long.toString(System.currentTimeMillis()));
 261
 262             // options
 263             if(options != null) {
 264                 if(options.get("timeout") != null && options.get("timeout") instanceof String) {
 265                     conn.setRequestProperty(RestMessageHeaders.X_VOLD_REQUEST_TIMEOUT_MS,
 266                                             (String) options.get("timeout"));
 267                 }
 268                 if(options.get("responseCode") != null
 269                    && options.get("responseCode") instanceof Integer) {
 270                     expectedResponseCode = (Integer) options.get("responseCode");
 271                 }
 272             }
 273
 274             // Check for the right response code
 275             if(conn.getResponseCode() != expectedResponseCode) {
 276                 System.err.println("Illegal response during DELETE : " + conn.getResponseMessage());
 277                 fail("Incorrect response received for a HTTP put request :"
 278                      + conn.getResponseCode());
 279             } else {
 280                 return true;
 281             }
 282
 283         } catch(Exception e) {
 284             e.printStackTrace();
 285             fail("Error in sending the REST request");
 286         }
 287
 288         return false;
 289     }
 290
 291     private TestVersionedValue doGet(String key) {
 292         return doGet(key, null);
 293     }
 294
 295     private TestVersionedValue doGet(String key, Map<String, Object> options) {
 296         HttpURLConnection conn = null;
 297         String response = null;
 298         TestVersionedValue responseObj = null;
 299         int expectedResponseCode = 200;
 300         try {
 301
 302             // Create the right URL and Http connection
 303             String base64Key = new String(Base64.encodeBase64(key.getBytes()));
 304             URL url = new URL(this.coordinatorURL + "/" + STORE_NAME + "/" + base64Key);
 305             conn = (HttpURLConnection) url.openConnection();
 306
 307             // Set the right headers
 308             conn.setRequestMethod("GET");
 309             conn.setDoInput(true);
 310             conn.setRequestProperty(RestMessageHeaders.X_VOLD_REQUEST_TIMEOUT_MS, "1000");
 311             conn.setRequestProperty(RestMessageHeaders.X_VOLD_REQUEST_ORIGIN_TIME_MS,
 312                                     Long.toString(System.currentTimeMillis()));
 313
 314             // options
 315             if(options != null) {
 316                 if(options.get("timeout") != null && options.get("timeout") instanceof String) {
 317                     conn.setRequestProperty(RestMessageHeaders.X_VOLD_REQUEST_TIMEOUT_MS,
 318                                             (String) options.get("timeout"));
 319                 }
 320                 if(options.get("responseCode") != null
 321                    && options.get("responseCode") instanceof Integer) {
 322                     expectedResponseCode = (Integer) options.get("responseCode");
 323                 }
 324             }
 325
 326             // Check for the right response code
 327             if(conn.getResponseCode() != expectedResponseCode) {
 328                 System.err.println("Illegal response during GET : " + conn.getResponseMessage());
 329                 fail("Incorrect response received for a HTTP GET request :"
 330                      + conn.getResponseCode());
 331             }
 332
 333             if(conn.getResponseCode() == 404 || conn.getResponseCode() == 408) {
 334                 return null;
 335             }
 336
 337             // Buffer the result into a string
 338             ByteArrayDataSource ds = new ByteArrayDataSource(conn.getInputStream(),
 339                                                              "multipart/mixed");
 340             MimeMultipart mp = new MimeMultipart(ds);
 341             assertEquals("The number of body parts expected is not 1", 1, mp.getCount());
 342
 343             MimeBodyPart part = (MimeBodyPart) mp.getBodyPart(0);
 344             VectorClock vc = RestUtils.deserializeVectorClock(part.getHeader(RestMessageHeaders.X_VOLD_VECTOR_CLOCK)[0]);
 345             int contentLength = Integer.parseInt(part.getHeader(RestMessageHeaders.CONTENT_LENGTH)[0]);
 346             byte[] bodyPartBytes = new byte[contentLength];
 347
 348             part.getInputStream().read(bodyPartBytes);
 349             response = new String(bodyPartBytes);
 350
 351             responseObj = new TestVersionedValue(response, vc);
 352
 353         } catch(Exception e) {
 354             e.printStackTrace();
 355             fail("Error in sending the REST request");
 356         } finally {
 357             if(conn != null) {
 358                 conn.disconnect();
 359             }
 360         }
 361         return responseObj;
 362     }
 363
 364     @Test
 365     public void testReadAfterWrite() {
 366         String key = "Which_Imperial_IPA_do_I_want_to_drink";
 367         String payload = "Pliny the Younger";
 368
 369         // 1. Do a put
 370         doPut(key, payload, null);
 371
 372         // 2. Do a get on the same key
 373         TestVersionedValue response = doGet(key);
 374         if(response == null) {
 375             fail("key does not exist after a put. ");
 376         }
 377
 378         System.out.println("Received value: " + response.getValue());
 379         if(!response.getValue().equals(payload)) {
 380             fail("Received value is incorrect ! Expected : " + payload + " but got : "
 381                  + response.getValue());
 382         }
 383     }
 384
 385     @Test
 386     public void testDelete() {
 387         String key = "Which_sour_beer_do_I_want_to_drink";
 388         String payload = "Duchesse De Bourgogne";
 389         Map<String, Object> options = new HashMap<String, Object>();
 390
 391         // 1. Do a put
 392         doPut(key, payload, null);
 393
 394         // 2. Do a get on the same key
 395         TestVersionedValue response = doGet(key);
 396         if(response == null) {
 397             fail("key does not exist after a put. ");
 398         }
 399         System.out.println("Received value: " + response.getValue());
 400         if(!response.getValue().equals(payload)) {
 401             fail("Received value is incorrect ! Expected : " + payload + " but got : "
 402                  + response.getValue());
 403         }
 404
 405         // 3. Do a delete
 406         boolean isDeleted = doDelete(key);
 407         if(!isDeleted) {
 408             fail("Could not delete the key. Error !");
 409         }
 410
 411         // 4. Do a get on the same key : this should fail
 412         options.put("responseCode", 404);
 413         response = doGet(key, options);
 414         if(response != null) {
 415             fail("key still exists after deletion. ");
 416         }
 417     }
 418
 419     @Test
 420     public void testVersionedPut() {
 421         String key = "Which_Porter_do_I_want_to_drink";
 422         String payload = "Founders Porter";
 423         String newPayload = "Samuel Smith Taddy Porter";
 424
 425         // 1. Do a put
 426         doPut(key, payload, null);
 427
 428         // 2. Do a get on the same key
 429         TestVersionedValue response = doGet(key, null);
 430         if(response == null) {
 431             fail("key does not exist after a put. ");
 432         }
 433         System.out.println("Received value: " + response.getValue());
 434
 435         // 3. Do a versioned put based on the version received previously
 436         doPut(key, newPayload, response.getVc());
 437
 438         // 4. Do a get again on the same key
 439         TestVersionedValue newResponse = doGet(key);
 440         if(newResponse == null) {
 441             fail("key does not exist after the versioned put. ");
 442         }
 443         assertEquals("Returned response does not have a higer version",
 444                      Occurred.AFTER,
 445                      newResponse.getVc().compare(response.getVc()));
 446         assertEquals("Returned response does not have a higer version",
 447                      Occurred.BEFORE,
 448                      response.getVc().compare(newResponse.getVc()));
 449
 450         System.out.println("Received value after the Versioned put: " + newResponse.getValue());
 451         if(!newResponse.getValue().equals(newPayload)) {
 452             fail("Received value is incorrect ! Expected : " + newPayload + " but got : "
 453                  + newResponse.getValue());
 454         }
 455     }
 456
 457     @Test
 458     public void testLargeValueSizeVersionedPut() {
 459         String key = "amigo";
 460         String payload = generateRandomString(new CoordinatorConfig().getHttpMessageDecoderMaxChunkSize() * 10,
 461                                               ValueType.ALPHA);
 462         String newPayload = generateRandomString(new CoordinatorConfig().getHttpMessageDecoderMaxChunkSize() * 10,
 463                                                  ValueType.ALPHA);
 464
 465         // 1. Do a put
 466         doPut(key, payload, null);
 467
 468         // 2. Do a get on the same key
 469         TestVersionedValue response = doGet(key, null);
 470         if(response == null) {
 471             fail("key does not exist after a put. ");
 472         }
 473         System.out.println("Received value: " + response.getValue());
 474
 475         // 3. Do a versioned put based on the version received previously
 476         doPut(key, newPayload, response.getVc());
 477
 478         // 4. Do a get again on the same key
 479         TestVersionedValue newResponse = doGet(key);
 480         if(newResponse == null) {
 481             fail("key does not exist after the versioned put. ");
 482         }
 483         assertEquals("Returned response does not have a higer version",
 484                      Occurred.AFTER,
 485                      newResponse.getVc().compare(response.getVc()));
 486         assertEquals("Returned response does not have a higer version",
 487                      Occurred.BEFORE,
 488                      response.getVc().compare(newResponse.getVc()));
 489
 490         System.out.println("Received value after the Versioned put: " + newResponse.getValue());
 491         if(!newResponse.getValue().equals(newPayload)) {
 492             fail("Received value is incorrect ! Expected : " + newPayload + " but got : "
 493                  + newResponse.getValue());
 494         }
 495     }
 496
 497     @Test
 498     public void testWriteWithTimeout() {
 499         String key = "Which_Imperial_IPA_do_I_want_to_drink";
 500         String payload = "Pliny the Younger";
 501         Map<String, Object> options = new HashMap<String, Object>();
 502
 503         // 1. Do a put (timeout)
 504         options.put("timeout", "1");
 505         options.put("responseCode", 408);
 506         doPut(key, payload, null, options);
 507
 508         // 2. Do a get on the same key
 509         options.clear();
 510         options.put("responseCode", 404);
 511         TestVersionedValue response = doGet(key, options);
 512         if(response != null) {
 513             fail("key should not exist after a put. ");
 514         }
 515
 516         // 3. Do a put
 517         doPut(key, payload, null);
 518
 519         // 4. Do a get on the same key with timeout
 520         options.clear();
 521         options.put("timeout", "1");
 522         options.put("responseCode", 408);
 523         response = doGet(key, options);
 524
 525         // 5. Do a get on the same key
 526         response = doGet(key);
 527         if(response == null) {
 528             fail("key does not exist after a put. ");
 529         }
 530     }
 531 }