Add reseeding command to bootstrap tool and fix some bugs related to reseeding. Add unit test

src/main/java/org/sdnplatform/sync/client/AuthTool.java

@@ -13,21 +13,22 @@ import org.sdnplatform.sync.internal.util.CryptoUtil;
  */
 public class AuthTool {
     protected static class AuthToolSettings {
-        @Option(name="--help", 
+        @Option(name="--help", aliases="-h",
                 usage="Show help")
         protected boolean help;
         
-        @Option(name="--keyStorePath", 
+        @Option(name="--keyStorePath", aliases="-ks",
                 usage="Path to JCEKS key store where credentials should " + 
                        "be stored")
         protected String keyStorePath;
         
-        @Option(name="--keyStorePassword",
+        @Option(name="--keyStorePassword", aliases="-kp",
                 usage="Password for key store")
         protected String keyStorePassword;    
 
-        @Option(name="--authScheme",
-                usage="Auth scheme for which we should set up credentials")
+        @Option(name="--authScheme", aliases="-a",
+                usage="Auth scheme for which we should set up credentials " +
+                      "(default NO_AUTH)")
         protected AuthScheme authScheme = AuthScheme.NO_AUTH;
 
         CmdLineParser parser = new CmdLineParser(this);

src/main/java/org/sdnplatform/sync/client/BootstrapTool.java

@@ -48,6 +48,14 @@ public class BootstrapTool extends SyncClientBase {
                 usage="Set the hostname for the local node (overrides " + 
                       "localNodeIface)")
         protected String localNodeHost;
+        
+        @Option(name="--reseed", aliases="-r",
+                usage="If you simultaneously change the IP of every node " + 
+                      "in the cluster, the cluster may not automatically " +
+                      "reform.  Run this command to cause it to rerun the " +
+                      "bootstrap process while retaining existing node IDs. " + 
+                      "The node will be put into its own local domain.")
+        protected boolean reseed;
     }
 
     public BootstrapTool(BootstrapToolSettings bootstrapSettings) {
@@ -64,6 +72,22 @@ public class BootstrapTool extends SyncClientBase {
                 syncManager.getStoreClient(SyncStoreCCProvider.
                                            SYSTEM_NODE_STORE, 
                                            Short.class, Node.class);
+        Short localNodeId = null;
+        if (bSettings.reseed || bSettings.domainId != 0) {
+            String localNodeIdStr = 
+                    waitForValue(uStoreClient, 
+                                 SyncStoreCCProvider.LOCAL_NODE_ID, 
+                                 5000000);
+            if (localNodeIdStr == null) {
+                err.println("Error: Local node ID is not set; you must " + 
+                            "first seed the cluster by using the --seeds " + 
+                            "option");
+                System.exit(3);
+            }
+            localNodeId = Short.valueOf(localNodeIdStr);
+        }
+        
+        
         if (bSettings.localNodeIface != null) {
             while (true) {
                 try {
@@ -91,6 +115,14 @@ public class BootstrapTool extends SyncClientBase {
                 } catch (ObsoleteVersionException e) {}
             }
         }
+        if (bSettings.reseed) {
+            while (true) {
+                try {
+                    nodeStoreClient.delete(localNodeId);
+                    break;
+                } catch (ObsoleteVersionException e) { };
+            }
+        }
         if (bSettings.seeds != null) {
             String[] seedsStr = bSettings.seeds.split(",");
             boolean seedsvalid = true;
@@ -122,17 +154,6 @@ public class BootstrapTool extends SyncClientBase {
             }
         }
         if (bSettings.domainId != 0) {
-            String localNodeIdStr = 
-                    waitForValue(uStoreClient, 
-                                 SyncStoreCCProvider.LOCAL_NODE_ID, 
-                                 5000000);
-            if (localNodeIdStr == null) {
-                err.println("Could not set domain ID for local node because " +
-                            "local node ID is not set");
-                System.exit(3);
-            }
-            Short localNodeId = Short.valueOf(localNodeIdStr);
-
             while (true) {
                 try {
                     Versioned<Node> localNode = 

src/main/java/org/sdnplatform/sync/client/SyncClientBase.java

@@ -44,11 +44,12 @@ public abstract class SyncClientBase {
      */
     protected static class SyncClientBaseSettings 
         extends AuthTool.AuthToolSettings {
-        @Option(name="--hostname", aliases="-h", 
-                usage="Server hostname")
+        @Option(name="--hostname", aliases="-n", 
+                usage="Server hostname (default \"localhost\")")
         protected String hostname = "localhost";
     
-        @Option(name="--port", aliases="-p", usage="Server port")
+        @Option(name="--port", aliases="-p", 
+                usage="Server port (default 6642)")
         protected int port = 6642;
         
         @Override

src/main/java/org/sdnplatform/sync/internal/SyncManager.java

@@ -590,10 +590,10 @@ public class SyncManager extends AbstractSyncManager {
 
     @LogMessageDocs({
         @LogMessageDoc(level="INFO",
-                message="Updating sync configuration {config}",
+                message="[{id}] Updating sync configuration {config}",
                 explanation="The sync service cluster configuration has been updated"),
         @LogMessageDoc(level="INFO",
-                message="Local node configuration changed; restarting sync" +
+                message="[{id}] Local node configuration changed; restarting sync" +
                         "service",
                 explanation="The sync service must be restarted to update its configuration")
     })
@@ -605,11 +605,13 @@ public class SyncManager extends AbstractSyncManager {
             clusterConfig = clusterConfigProvider.getConfig();
             if (clusterConfig.equals(oldConfig)) return;
 
-            logger.info("Updating sync configuration {}", clusterConfig);
+            logger.info("[{}] Updating sync configuration {}", 
+                        clusterConfig.getNode().getNodeId(),
+                        clusterConfig);
             if (oldConfig.getNode() != null &&
                 !clusterConfig.getNode().equals(oldConfig.getNode())) {
-                logger.info("Local node configuration changed; restarting sync" +
-                        "service");
+                logger.info("[{}] Local node configuration changed; restarting sync" +
+                        "service", oldConfig.getNode().getNodeId());
                 shutdown();
                 startUp(null);
             }

src/main/java/org/sdnplatform/sync/internal/config/DelegatingCCProvider.java

@@ -48,6 +48,9 @@ public class DelegatingCCProvider implements IClusterConfigProvider {
         for (IClusterConfigProvider provider : providers) {
             try {
                 return provider.getConfig();
+            } catch (RuntimeException e) {
+                logger.debug("RuntimeException in ClusterConfig provider {}",
+                             provider.getClass().getSimpleName(), e);
             } catch (Exception e) {
                 logger.debug("ClusterConfig provider {} failed: {}",
                              provider.getClass().getSimpleName(),

src/main/java/org/sdnplatform/sync/internal/config/SyncStoreCCProvider.java

@@ -165,14 +165,16 @@ public class SyncStoreCCProvider
                     break;
                 } catch (ObsoleteVersionException e) { }
             }
+            if (newLocalNode == null) {
+                newLocalNode = getLocalNode(localNodeId, localNodeId);
+            }
             nodes.add(newLocalNode);
             
             if (oldLocalNode == null || !oldLocalNode.equals(newLocalNode)) {
                 // If we have no local node or our hostname or port changes, 
                 // we should trigger a new cluster join to ensure that the 
                 // new value can propagate everywhere
-                bootstrapTask.reschedule(0, TimeUnit.SECONDS);
-                throw new SyncException("Local node configuration has changed");
+                bootstrapTask.reschedule(0, TimeUnit.SECONDS);    
             }
             
             ClusterConfig config = new ClusterConfig(nodes, localNodeId,
@@ -221,8 +223,9 @@ public class SyncStoreCCProvider
                 Versioned<String> sv = unsyncStoreClient.get(SEEDS);
                 if (sv.getValue() == null || !sv.getValue().equals(seeds)) {
                     if (logger.isDebugEnabled()) {
-                        logger.debug("Updating seeds to \"{}\" from \"{}\"", 
-                                     seeds, sv.getValue());
+                        logger.debug("[{}] Updating seeds to \"{}\" from \"{}\"", 
+                                     new Object[]{config.getNode().getNodeId(),
+                                                  seeds, sv.getValue()});
                     }
                     unsyncStoreClient.put(SEEDS, seeds);
                 }
@@ -286,10 +289,11 @@ public class SyncStoreCCProvider
     protected class BootstrapTask implements Runnable {
         @Override
         public void run() {
+            Short localNodeId = null;
             try {
                 Node localNode = null;
 
-                Short localNodeId = getLocalNodeId();
+                localNodeId = getLocalNodeId();
                 if (localNodeId != null)
                     localNode = nodeStoreClient.getValue(localNodeId);
 
@@ -301,7 +305,7 @@ public class SyncStoreCCProvider
                              localNodeId);
 
                 if (seedStr.equals("")) {
-                    localNode = setupLocalNode(localNode, true);
+                    localNode = setupLocalNode(localNode, localNodeId, true);
                     if (logger.isDebugEnabled()) {
                         logger.debug("[{}] First node configuration: {}",
                                      localNode.getNodeId(), localNode);
@@ -328,7 +332,7 @@ public class SyncStoreCCProvider
                                      localNode.getNodeId());
                     }
                 } else {
-                    localNode = setupLocalNode(localNode, false);
+                    localNode = setupLocalNode(localNode, localNodeId, false);
                     if (logger.isDebugEnabled()) {
                         logger.debug("[{}] Adding new node from seeds {}: {}", 
                                      new Object[]{localNodeId, seedStr, 
@@ -361,17 +365,21 @@ public class SyncStoreCCProvider
                 }
                 syncManager.updateConfiguration();
             } catch (Exception e) {
-                logger.error("Failed to bootstrap cluster", e);
+                logger.error("[" + localNodeId + 
+                             "] Failed to bootstrap cluster", e);
             }
         }
 
-        private Node setupLocalNode(Node localNode, boolean firstNode) 
+        private Node setupLocalNode(Node localNode, Short localNodeId,
+                                    boolean firstNode) 
                 throws SyncException {
             short nodeId = -1;
             short domainId = -1;
             if (localNode != null) {
                 nodeId = localNode.getNodeId();
                 domainId = localNode.getDomainId();
+            } else if (localNodeId != null) {
+                domainId = nodeId = localNodeId;
             } else if (firstNode) {
                 domainId = nodeId = 
                         (short)(new Random().nextInt(Short.MAX_VALUE));

src/main/java/org/sdnplatform/sync/internal/config/bootstrap/Bootstrap.java

@@ -120,7 +120,9 @@ public class Bootstrap {
             return false;
         }
         Channel channel = future.getChannel();
-        logger.debug("Connected to " + seed);
+        logger.debug("[{}] Connected to {}", 
+                     localNode != null ? localNode.getNodeId() : null,
+                     seed);
         
         try {
             channel.getCloseFuture().await();

src/main/java/org/sdnplatform/sync/internal/config/bootstrap/BootstrapChannelHandler.java

@@ -31,7 +31,6 @@ public class BootstrapChannelHandler extends AbstractRPCChannelHandler {
             LoggerFactory.getLogger(BootstrapChannelHandler.class);
 
     private Bootstrap bootstrap;
-    private Short localNodeId;
     private Short remoteNodeId;
     
     public BootstrapChannelHandler(Bootstrap bootstrap) {
@@ -65,7 +64,7 @@ public class BootstrapChannelHandler extends AbstractRPCChannelHandler {
             n.setNodeId(bootstrap.localNode.getNodeId());
         if (bootstrap.localNode.getDomainId() >= 0)
             n.setDomainId(bootstrap.localNode.getDomainId());
-
+        
         ClusterJoinRequestMessage cjrm = new ClusterJoinRequestMessage();
         AsyncMessageHeader header = new AsyncMessageHeader();
         header.setTransactionId(bootstrap.transactionId.getAndIncrement());
@@ -131,12 +130,12 @@ public class BootstrapChannelHandler extends AbstractRPCChannelHandler {
 
     @Override
     protected Short getRemoteNodeId() {
-        return localNodeId;
+        return remoteNodeId;
     }
 
     @Override
     protected Short getLocalNodeId() {
-        return remoteNodeId;
+        return null;
     }
 
     @Override

src/test/java/org/sdnplatform/sync/internal/BootstrapTest.java

@@ -137,5 +137,17 @@ public class BootstrapTest {
         SyncManager[] syncManagerArr = 
                 syncManagers.toArray(new SyncManager[syncManagers.size()]);
         waitForFullMesh(syncManagerArr, 5000);
+        
+        logger.info("Cluster successfully build.  Attempting reseed");
+        
+        // Test reseeding
+        nodeStores.get(0).delete(nodeIds.get(0));
+        
+        for (int j = 0; j < nodeIds.size(); j++) {
+            for (int k = 0; k < nodeIds.size(); k++) {
+                waitForValue(nodeStores.get(j), nodeIds.get(k), 
+                             nodes.get(k), 3000, "nodeStore" + j);
+            }
+        }
     }
 }