From 47e9ac8abf534fee8bb03f2651f304eb33b56dc2 Mon Sep 17 00:00:00 2001
From: Zhongjie Wu <www.zj30@gmail.com>
Date: Tue, 17 Dec 2013 15:44:15 -0800
Subject: [PATCH] Fix for problems on metadata change with less nodes

1. Check for node existence in InvalidMetadataCheckingStore and
throw InvalidMetadataException if specified node is not in the cluster
2. Have ZenStoreClient rebootstrap when getting InvalidMetadataException.
This fixes if the new cluster does not have a node that ZenStoreClient query
data from, it would throw Exception.
---
 .../client/scheduler/AsyncMetadataVersionManager.java       |  7 +++++++
 src/java/voldemort/store/StoreUtils.java                    | 13 +++++++++++++
 .../store/invalidmetadata/InvalidMetadataCheckingStore.java |  4 ++++
 3 files changed, 24 insertions(+)

diff --git a/src/java/voldemort/client/scheduler/AsyncMetadataVersionManager.java b/src/java/voldemort/client/scheduler/AsyncMetadataVersionManager.java
index ea73c3115..35dec4ba4 100644
--- a/src/java/voldemort/client/scheduler/AsyncMetadataVersionManager.java
+++ b/src/java/voldemort/client/scheduler/AsyncMetadataVersionManager.java
@@ -22,6 +22,7 @@ import java.util.concurrent.Callable;
 import org.apache.log4j.Logger;
 
 import voldemort.client.SystemStoreRepository;
+import voldemort.store.InvalidMetadataException;
 import voldemort.utils.MetadataVersionStoreUtils;
 
 /**
@@ -177,6 +178,12 @@ public class AsyncMetadataVersionManager implements Runnable {
                 }
             }
 
+        } catch(InvalidMetadataException e) {
+            try {
+                this.storeClientThunk.call();
+            } catch(Exception e2) {
+                logger.error("Exception occurred while invoking the rebootstrap callback.", e);
+            }
         } catch(Exception e) {
             if(logger.isDebugEnabled()) {
                 logger.debug("Could not retrieve metadata versions from the server.", e);
diff --git a/src/java/voldemort/store/StoreUtils.java b/src/java/voldemort/store/StoreUtils.java
index 7cb2cdb6b..f79e586f6 100644
--- a/src/java/voldemort/store/StoreUtils.java
+++ b/src/java/voldemort/store/StoreUtils.java
@@ -32,6 +32,7 @@ import voldemort.routing.RoutingStrategy;
 import voldemort.serialization.Serializer;
 import voldemort.serialization.SerializerDefinition;
 import voldemort.serialization.SerializerFactory;
+import voldemort.store.metadata.MetadataStore;
 import voldemort.utils.ByteArray;
 import voldemort.utils.ClosableIterator;
 import voldemort.utils.Pair;
@@ -144,6 +145,18 @@ public class StoreUtils {
                                            + " not present at " + currentNode);
     }
 
+    /**
+     * Check if the the nodeId is present in the cluster managed by the metadata store
+     * or throw an exception.
+     *
+     * @param nodeId The nodeId to check existence of
+     */
+    public static void assertValidNode(MetadataStore metadataStore, Integer nodeId) {
+        if(!metadataStore.getCluster().hasNodeWithId(nodeId)) {
+            throw new InvalidMetadataException("NodeId " + nodeId + " is not or no longer in this cluster");
+        }
+    }
+
     public static <V> List<Version> getVersions(List<Versioned<V>> versioneds) {
         List<Version> versions = Lists.newArrayListWithCapacity(versioneds.size());
         for(Versioned<?> versioned: versioneds)
diff --git a/src/java/voldemort/store/invalidmetadata/InvalidMetadataCheckingStore.java b/src/java/voldemort/store/invalidmetadata/InvalidMetadataCheckingStore.java
index c4d126f43..c08bf7d07 100644
--- a/src/java/voldemort/store/invalidmetadata/InvalidMetadataCheckingStore.java
+++ b/src/java/voldemort/store/invalidmetadata/InvalidMetadataCheckingStore.java
@@ -64,6 +64,7 @@ public class InvalidMetadataCheckingStore extends DelegatingStore<ByteArray, byt
     @Override
     public boolean delete(ByteArray key, Version version) throws VoldemortException {
         StoreUtils.assertValidKey(key);
+        StoreUtils.assertValidNode(metadata, nodeId);
         StoreUtils.assertValidMetadata(key,
                                        metadata.getRoutingStrategy(getName()),
                                        metadata.getCluster().getNodeById(nodeId));
@@ -76,6 +77,7 @@ public class InvalidMetadataCheckingStore extends DelegatingStore<ByteArray, byt
                                                           Map<ByteArray, byte[]> transforms)
             throws VoldemortException {
         StoreUtils.assertValidKeys(keys);
+        StoreUtils.assertValidNode(metadata, nodeId);
         RoutingStrategy routingStrategy = metadata.getRoutingStrategy(getName());
         Node node = metadata.getCluster().getNodeById(nodeId);
         for(ByteArray key: keys)
@@ -88,6 +90,7 @@ public class InvalidMetadataCheckingStore extends DelegatingStore<ByteArray, byt
     public void put(ByteArray key, Versioned<byte[]> value, byte[] transforms)
             throws VoldemortException {
         StoreUtils.assertValidKey(key);
+        StoreUtils.assertValidNode(metadata, nodeId);
         StoreUtils.assertValidMetadata(key,
                                        metadata.getRoutingStrategy(getName()),
                                        metadata.getCluster().getNodeById(nodeId));
@@ -98,6 +101,7 @@ public class InvalidMetadataCheckingStore extends DelegatingStore<ByteArray, byt
     @Override
     public List<Versioned<byte[]>> get(ByteArray key, byte[] transforms) throws VoldemortException {
         StoreUtils.assertValidKey(key);
+        StoreUtils.assertValidNode(metadata, nodeId);
         StoreUtils.assertValidMetadata(key,
                                        metadata.getRoutingStrategy(getName()),
                                        metadata.getCluster().getNodeById(nodeId));
-- 
2.11.4.GIT